- Action schema is optimized taking into account device width

- Using fewer CUDA streams.

nnforge/absolute_layer.cpp

@@ -34,15 +34,18 @@ namespace nnforge
 		return layer::ptr(new absolute_layer(*this));
 	}
 
-	float absolute_layer::get_forward_flops(const std::vector<layer_configuration_specific>& input_configuration_specific_list) const
-	{
-		return static_cast<float>(input_configuration_specific_list[0].get_neuron_count());
-	}
-
-	float absolute_layer::get_backward_flops(
+	float absolute_layer::get_flops_per_entry(
 		const std::vector<layer_configuration_specific>& input_configuration_specific_list,
-		unsigned int input_layer_id) const
+		const layer_action& action) const
 	{
-		return static_cast<float>(input_configuration_specific_list[0].get_neuron_count());
+		switch (action.get_action_type())
+		{
+		case layer_action::forward:
+			return static_cast<float>(input_configuration_specific_list[0].get_neuron_count());
+		case layer_action::backward_data:
+			return static_cast<float>(input_configuration_specific_list[0].get_neuron_count());
+		default:
+			return 0.0F;
+		}
 	}
 }

nnforge/absolute_layer.h

@@ -28,11 +28,9 @@ namespace nnforge
 
 		virtual layer::ptr clone() const;
 
-		virtual float get_forward_flops(const std::vector<layer_configuration_specific>& input_configuration_specific_list) const;
-
-		virtual float get_backward_flops(
+		virtual float get_flops_per_entry(
 			const std::vector<layer_configuration_specific>& input_configuration_specific_list,
-			unsigned int input_layer_id) const;
+			const layer_action& action) const;
 
 		virtual std::string get_type_name() const;
 

nnforge/accuracy_layer.cpp

@@ -92,19 +92,23 @@ namespace nnforge
 		}
 	}
 
-	float accuracy_layer::get_forward_flops(const std::vector<layer_configuration_specific>& input_configuration_specific_list) const
-	{
-		unsigned int neuron_count = get_output_layer_configuration_specific(input_configuration_specific_list).get_neuron_count();
-		unsigned int per_item_flops = input_configuration_specific_list[0].feature_map_count * 2;
-
-		return static_cast<float>(neuron_count) * static_cast<float>(per_item_flops);
-	}
-
-	float accuracy_layer::get_backward_flops(
+	float accuracy_layer::get_flops_per_entry(
 		const std::vector<layer_configuration_specific>& input_configuration_specific_list,
-		unsigned int input_layer_id) const
+		const layer_action& action) const
 	{
-		throw neural_network_exception("get_backward_flops is not implemented for accuracy_layer");
+		switch (action.get_action_type())
+		{
+		case layer_action::forward:
+			{
+				unsigned int neuron_count = get_output_layer_configuration_specific(input_configuration_specific_list).get_neuron_count();
+				unsigned int per_item_flops = input_configuration_specific_list[0].feature_map_count * 2;
+				return static_cast<float>(neuron_count) * static_cast<float>(per_item_flops);
+			}
+		case layer_action::backward_data:
+			throw neural_network_exception("get_backward_flops is not implemented for accuracy_layer");
+		default:
+			return 0.0F;
+		}
 	}
 
 	std::string accuracy_layer::get_string_for_average_data(

nnforge/accuracy_layer.h

@@ -38,11 +38,9 @@ namespace nnforge
 			const layer_configuration_specific& output_configuration_specific,
 			unsigned int input_layer_id) const;
 
-		virtual float get_forward_flops(const std::vector<layer_configuration_specific>& input_configuration_specific_list) const;
-
-		virtual float get_backward_flops(
+		virtual float get_flops_per_entry(
 			const std::vector<layer_configuration_specific>& input_configuration_specific_list,
-			unsigned int input_layer_id) const;
+			const layer_action& action) const;
 
 		virtual std::string get_type_name() const;
 

nnforge/average_subsampling_layer.cpp

@@ -140,21 +140,26 @@ namespace nnforge
 		check();
 	}
 
-	float average_subsampling_layer::get_forward_flops(const std::vector<layer_configuration_specific>& input_configuration_specific_list) const
-	{
-		unsigned int neuron_count = get_output_layer_configuration_specific(input_configuration_specific_list).get_neuron_count();
-		unsigned int per_item_flops = 1;
-		std::for_each(subsampling_sizes.begin(), subsampling_sizes.end(), per_item_flops *= boost::lambda::_1);
-
-		return static_cast<float>(neuron_count) * static_cast<float>(per_item_flops);
-	}
-
-	float average_subsampling_layer::get_backward_flops(
+	float average_subsampling_layer::get_flops_per_entry(
 		const std::vector<layer_configuration_specific>& input_configuration_specific_list,
-		unsigned int input_layer_id) const
+		const layer_action& action) const
 	{
-		unsigned int neuron_count = get_output_layer_configuration_specific(input_configuration_specific_list).get_neuron_count();
-
-		return static_cast<float>(neuron_count);
+		switch (action.get_action_type())
+		{
+		case layer_action::forward:
+			{
+				unsigned int neuron_count = get_output_layer_configuration_specific(input_configuration_specific_list).get_neuron_count();
+				unsigned int per_item_flops = 1;
+				std::for_each(subsampling_sizes.begin(), subsampling_sizes.end(), per_item_flops *= boost::lambda::_1);
+				return static_cast<float>(neuron_count) * static_cast<float>(per_item_flops);
+			}
+		case layer_action::backward_data:
+			{
+				unsigned int neuron_count = get_output_layer_configuration_specific(input_configuration_specific_list).get_neuron_count();
+				return static_cast<float>(neuron_count);
+			}
+		default:
+			return 0.0F;
+		}
 	}
 }

nnforge/average_subsampling_layer.h

@@ -43,11 +43,9 @@ namespace nnforge
 			const std::vector<std::pair<unsigned int, unsigned int> >& output_rectangle_borders,
 			unsigned int input_layer_id) const;
 
-		virtual float get_forward_flops(const std::vector<layer_configuration_specific>& input_configuration_specific_list) const;
-
-		virtual float get_backward_flops(
+		virtual float get_flops_per_entry(
 			const std::vector<layer_configuration_specific>& input_configuration_specific_list,
-			unsigned int input_layer_id) const;
+			const layer_action& action) const;
 
 		virtual std::string get_type_name() const;
 

nnforge/backward_propagation.cpp

@@ -81,7 +81,7 @@ namespace nnforge
 		}
 		if (debug->is_debug())
 		{
-			std::vector<layer_name_with_action> actions = action_schema->get_actions_in_execution_order();
+			std::vector<layer_name_with_action> actions = action_schema->get_actions();
 			std::map<layer_name_with_action, unsigned int> layer_name_with_action_color_map;
 			for(std::vector<layer_name_with_action>::const_iterator it = actions.begin(); it != actions.end(); ++it)
 			{
@@ -149,7 +149,7 @@ namespace nnforge
 
 	void backward_propagation::update_flops()
 	{
-		flops = action_schema->get_flops(layer_config_map);
+		flops = action_schema->get_flops(layer_config_map, std::map<std::string, unsigned int>());
 	}
 
 	backward_propagation::stat backward_propagation::run(

nnforge/convolution_layer.cpp

@@ -277,34 +277,24 @@ namespace nnforge
 		std::fill((*data)[1].begin(), (*data)[1].end(), 0.0F);
 	}
 
-	float convolution_layer::get_forward_flops(const std::vector<layer_configuration_specific>& input_configuration_specific_list) const
-	{
-		unsigned int neuron_count = get_output_layer_configuration_specific(input_configuration_specific_list).get_neuron_count();
-		unsigned int per_item_flops = input_feature_map_count * 2;
-		std::for_each(window_sizes.begin(), window_sizes.end(), per_item_flops *= boost::lambda::_1);
-		per_item_flops -= 1;
-
-		return static_cast<float>(neuron_count) * static_cast<float>(per_item_flops);
-	}
-
-	float convolution_layer::get_backward_flops(
+	float convolution_layer::get_flops_per_entry(
 		const std::vector<layer_configuration_specific>& input_configuration_specific_list,
-		unsigned int input_layer_id) const
-	{
-		unsigned int neuron_count = get_output_layer_configuration_specific(input_configuration_specific_list).get_neuron_count();
-		unsigned int per_item_flops = input_feature_map_count * 2;
-		std::for_each(window_sizes.begin(), window_sizes.end(), per_item_flops *= boost::lambda::_1);
-
-		return static_cast<float>(neuron_count) * static_cast<float>(per_item_flops);
-	}
-
-	float convolution_layer::get_weights_update_flops(const std::vector<layer_configuration_specific>& input_configuration_specific_list) const
+		const layer_action& action) const
 	{
-		unsigned int neuron_count = get_output_layer_configuration_specific(input_configuration_specific_list).get_neuron_count();
-		unsigned int per_item_flops = input_feature_map_count * 2;
-		std::for_each(window_sizes.begin(), window_sizes.end(), per_item_flops *= boost::lambda::_1);
-
-		return static_cast<float>(neuron_count) * static_cast<float>(per_item_flops);
+		switch (action.get_action_type())
+		{
+		case layer_action::forward:
+		case layer_action::backward_data:
+		case layer_action::backward_weights:
+			{
+				unsigned int neuron_count = get_output_layer_configuration_specific(input_configuration_specific_list).get_neuron_count();
+				unsigned int per_item_flops = input_feature_map_count * 2;
+				std::for_each(window_sizes.begin(), window_sizes.end(), per_item_flops *= boost::lambda::_1);
+				return static_cast<float>(neuron_count) * static_cast<float>(per_item_flops);
+			}
+		default:
+			return 0.0F;
+		}
 	}
 
 	layer_data_configuration_list convolution_layer::get_layer_data_configuration_list() const

nnforge/convolution_layer.h

@@ -49,13 +49,9 @@ namespace nnforge
 
 		virtual layer_data_configuration_list get_layer_data_configuration_list() const;
 
-		virtual float get_forward_flops(const std::vector<layer_configuration_specific>& input_configuration_specific_list) const;
-
-		virtual float get_backward_flops(
+		virtual float get_flops_per_entry(
 			const std::vector<layer_configuration_specific>& input_configuration_specific_list,
-			unsigned int input_layer_id) const;
-
-		virtual float get_weights_update_flops(const std::vector<layer_configuration_specific>& input_configuration_specific_list) const;
+			const layer_action& action) const;
 
 		virtual std::string get_type_name() const;
 

nnforge/cross_entropy_layer.cpp

@@ -91,21 +91,26 @@ namespace nnforge
 		}
 	}
 
-	float cross_entropy_layer::get_forward_flops(const std::vector<layer_configuration_specific>& input_configuration_specific_list) const
-	{
-		unsigned int neuron_count = get_output_layer_configuration_specific(input_configuration_specific_list).get_neuron_count();
-		unsigned int per_item_flops = input_configuration_specific_list[0].feature_map_count * 8;
-
-		return static_cast<float>(neuron_count) * static_cast<float>(per_item_flops);
-	}
-
-	float cross_entropy_layer::get_backward_flops(
+	float cross_entropy_layer::get_flops_per_entry(
 		const std::vector<layer_configuration_specific>& input_configuration_specific_list,
-		unsigned int input_layer_id) const
+		const layer_action& action) const
 	{
-		unsigned int neuron_count = input_configuration_specific_list[input_layer_id].get_neuron_count();
-		unsigned int per_item_flops = input_configuration_specific_list[input_layer_id].feature_map_count * 6;
-
-		return static_cast<float>(neuron_count) * static_cast<float>(per_item_flops);
+		switch (action.get_action_type())
+		{
+		case layer_action::forward:
+			{
+				unsigned int neuron_count = get_output_layer_configuration_specific(input_configuration_specific_list).get_neuron_count();
+				unsigned int per_item_flops = input_configuration_specific_list[0].feature_map_count * 8;
+				return static_cast<float>(neuron_count) * static_cast<float>(per_item_flops);
+			}
+		case layer_action::backward_data:
+			{
+				unsigned int neuron_count = input_configuration_specific_list[action.get_backprop_index()].get_neuron_count();
+				unsigned int per_item_flops = input_configuration_specific_list[action.get_backprop_index()].feature_map_count * 6;
+				return static_cast<float>(neuron_count) * static_cast<float>(per_item_flops);
+			}
+		default:
+			return 0.0F;
+		}
 	}
 }

nnforge/cross_entropy_layer.h

@@ -39,11 +39,9 @@ namespace nnforge
 			const layer_configuration_specific& output_configuration_specific,
 			unsigned int input_layer_id) const;
 
-		virtual float get_forward_flops(const std::vector<layer_configuration_specific>& input_configuration_specific_list) const;
-
-		virtual float get_backward_flops(
+		virtual float get_flops_per_entry(
 			const std::vector<layer_configuration_specific>& input_configuration_specific_list,
-			unsigned int input_layer_id) const;
+			const layer_action& action) const;
 
 		virtual std::string get_type_name() const;