From 4535b3d4722227eaddbc25d11476007fc85cd2b5 Mon Sep 17 00:00:00 2001 From: milakov Date: Sat, 29 Nov 2014 11:07:18 +0300 Subject: [PATCH] Fixed bug with dropout when error function is fuzed woth last activation function --- nnforge/cuda/network_updater_cuda.cu | 5 ++--- nnforge/network_updater.cpp | 5 +++++ nnforge/network_updater.h | 5 ++++- nnforge/neural_network_toolset.cpp | 5 ++++- nnforge/plain/network_updater_plain.cpp | 5 ++--- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/nnforge/cuda/network_updater_cuda.cu b/nnforge/cuda/network_updater_cuda.cu index 90cc366..6d07ff7 100644 --- a/nnforge/cuda/network_updater_cuda.cu +++ b/nnforge/cuda/network_updater_cuda.cu @@ -489,7 +489,6 @@ namespace nnforge cuda_safe_call(cudaEventQuery(data_processed_event)); } - random_generator gen = rnd::get_random_generator(); nnforge_uniform_int_distribution dist(0, static_cast(random_uniform_list.size() - 1)); unsigned int mask = static_cast(random_uniform_list.size() - 1); unsigned int entries_processed_count = 0; @@ -673,8 +672,8 @@ namespace nnforge std::vector >::reverse_iterator net_data_custom_it = net_data_custom.rbegin(); std::vector >::reverse_iterator gradient_it = gradient.rbegin(); std::vector >::reverse_iterator schema_data_it = updater_schema_data.rbegin(); - unsigned int reverse_layer_id = static_cast(updater_list.size() + testing_layer_count) - 1 - (error_function_fused_with_activation ? 1 : 0); - layer_configuration_specific_list::const_reverse_iterator layer_config_it = layer_config_list.rbegin() + 1; + unsigned int reverse_layer_id = static_cast(updater_list.size() + testing_layer_count) - 1; + layer_configuration_specific_list::const_reverse_iterator layer_config_it = layer_config_list.rbegin() + (1 + (error_function_fused_with_activation ? 1 : 0)); for(std::vector::reverse_iterator it = updater_list.rbegin(); it != updater_list.rend(); ++it, ++input_and_all_buffers_pack_it, ++schema_data_it, ++gradient_it, ++output_errors_it, ++net_data_it, ++net_data_custom_it, --reverse_layer_id, ++layer_config_it) { (*it)->enqueue_update_weights( diff --git a/nnforge/network_updater.cpp b/nnforge/network_updater.cpp index ce18c8c..15d892d 100644 --- a/nnforge/network_updater.cpp +++ b/nnforge/network_updater.cpp @@ -40,6 +40,11 @@ namespace nnforge { } + void network_updater::set_random_generator_seed(int seed) + { + gen = rnd::get_random_generator(seed); + } + void network_updater::set_input_configuration_specific(const layer_configuration_specific& input_configuration_specific) { if ((layer_config_list.size() > 0) && (layer_config_list[0] == input_configuration_specific)) diff --git a/nnforge/network_updater.h b/nnforge/network_updater.h index 0694a16..89c6825 100644 --- a/nnforge/network_updater.h +++ b/nnforge/network_updater.h @@ -50,6 +50,8 @@ namespace nnforge // set_input_configuration_specific should be called prior to this method call for this method to succeed float get_flops_for_single_entry() const; + void set_random_generator_seed(int seed); + protected: network_updater( network_schema_smart_ptr schema, @@ -78,12 +80,13 @@ namespace nnforge std::vector random_uniform_list; float flops; + random_generator gen; + private: network_updater(); network_updater(const network_updater&); network_updater& operator =(const network_updater&); - random_generator gen; static const unsigned int random_list_bits; }; diff --git a/nnforge/neural_network_toolset.cpp b/nnforge/neural_network_toolset.cpp index 53a6981..7bcfed6 100644 --- a/nnforge/neural_network_toolset.cpp +++ b/nnforge/neural_network_toolset.cpp @@ -1494,6 +1494,9 @@ namespace nnforge network_updater_smart_ptr updater = updater_factory->create( schema, get_error_function()); + // !!! Seeding to constant number will not guarantee that updater will run determnistically: + // Different updaters might use different internal batch sizes + updater->set_random_generator_seed(12349087); supervised_data_reader_smart_ptr training_data_reader = get_data_reader_for_training(true); training_data_reader = supervised_data_reader_smart_ptr(new supervised_limited_entry_count_data_reader(training_data_reader, profile_updater_entry_count)); @@ -1526,7 +1529,7 @@ namespace nnforge batch_size, weight_decay, momentum, - std::map()); + get_dropout_rate_map()); boost::chrono::duration sec = boost::chrono::high_resolution_clock::now() - start; /* diff --git a/nnforge/plain/network_updater_plain.cpp b/nnforge/plain/network_updater_plain.cpp index 9cfd765..2765f13 100644 --- a/nnforge/plain/network_updater_plain.cpp +++ b/nnforge/plain/network_updater_plain.cpp @@ -216,7 +216,6 @@ namespace nnforge } } - random_generator gen = rnd::get_random_generator(); nnforge_uniform_int_distribution dist(0, static_cast(random_uniform_list.size() - 1)); unsigned int mask = static_cast(random_uniform_list.size() - 1); bool entries_remained_for_loading = true; @@ -410,7 +409,7 @@ namespace nnforge layer_data_custom_list::const_reverse_iterator data_custom_it = data->data_custom_list.rbegin() + (error_function_fused_with_activation ? 1 : 0); layer_data_list::reverse_iterator gradient_it = gradient->rbegin() + (error_function_fused_with_activation ? 1 : 0); additional_buffer_smart_ptr output_errors = initial_error_buf; - unsigned int reverse_layer_id = static_cast(updater_list.size() + testing_layer_count) - 1 - (error_function_fused_with_activation ? 1 : 0); + unsigned int reverse_layer_id = static_cast(updater_list.size() + testing_layer_count) - 1; for(std::vector::const_reverse_iterator it = updater_list.rbegin(); it != updater_list.rend(); ++it, ++layer_it, ++input_config_it, ++updater_buffers_it, ++data_it, ++data_custom_it, ++gradient_it, --reverse_layer_id) { if (it != updater_list.rend() - 1) @@ -680,7 +679,7 @@ namespace nnforge float val = *(in_it + i); unsigned int random_elem_id = (i + offset_in_random_list) & mask; bool under_threshold = (*(rnd_it + random_elem_id) < dropout_rate); - val *= under_threshold ? 0.0F : scale; + val *= (under_threshold ? 0.0F : scale); *(in_it + i) = val; } }