ONNX fixes for 1.0

examples/CMakeLists.txt

@@ -145,6 +145,9 @@ if(BUILD_PROTOBUF)
     add_executable(onnx_gradients "onnx/4_onnx_test_gradients.cpp")
     target_link_libraries(onnx_gradients eddl)
 
+    add_executable(onnx_gradients_recurrent "onnx/7_onnx_test_gradients_recurrent.cpp")
+    target_link_libraries(onnx_gradients_recurrent eddl)
+
     add_executable(onnx_import_reshape "onnx/5_onnx_import_net_and_reshape.cpp")
     target_link_libraries(onnx_import_reshape eddl)
 
@@ -303,4 +306,4 @@ add_executable(test3 "test_internals/test3.cpp")
 target_link_libraries(test3 eddl)
 
 add_executable(test4 "test_internals/test4.cpp")
-target_link_libraries(test4 eddl)
+target_link_libraries(test4 eddl)

examples/applications/app_test_model.cpp

@@ -6,10 +6,10 @@
 using namespace eddl;
 
 
-Tensor* preprocess_input(Tensor* input, const vector<int> &target_size, bool normalize=true, bool standarize=true){
+Tensor* preprocess_input(Tensor* input, const vector<int> &target_size, const vector<float>& mean, const vector<float>& std, bool normalize=true, bool standarize=true, const string& channels_order="rgb"){
     // Define preprocessing constants
-    auto* mean_vec = new Tensor( {0.485, 0.456, 0.406}, {3}, input->device);
-    auto* std_vec = new Tensor( {0.229, 0.224, 0.225}, {3}, input->device);
+    auto* mean_vec = new Tensor(mean, {3}, input->device);
+    auto* std_vec = new Tensor( std, {3}, input->device);
 
     // ==========================================================================
     // ====== SANITY CHECKS =====================================================
@@ -51,6 +51,29 @@ Tensor* preprocess_input(Tensor* input, const vector<int> &target_size, bool nor
     }
     // ==========================================================================
 
+    // ==========================================================================
+    // ====== RE-ODER CHANNELS ==================================================
+    // ==========================================================================
+
+    if (channels_order == "bgr"){
+        // Take each channel of the image
+        Tensor *r_channel = new_input->select({":", "0", ":", ":"});
+        Tensor *g_channel = new_input->select({":", "1", ":", ":"});
+        Tensor *b_channel = new_input->select({":", "2", ":", ":"});
+
+        // Concat the channels reordering them
+        Tensor *bgr_input = Tensor::concat({b_channel, g_channel, r_channel}, 1);
+        Tensor::copy(bgr_input, new_input);
+
+        delete bgr_input;
+        delete r_channel;
+        delete g_channel;
+        delete b_channel;
+    }else{
+
+    }
+
+    // ==========================================================================
     // Free memory
     delete mean_vec;
     delete std_vec;
@@ -69,12 +92,12 @@ int main(int argc, char **argv) {
     string class_names_file = "../../examples/data/imagenet_class_names.txt";
 
     // Image Classification
-    string model_path = "models/resnet34-v1-7.onnx";  // 3x224x224  // okay
-//    string model_path = "models/mobilenetv2-7.onnx"; // 3x224x224 // Signal: SIGSEGV (Segmentation fault)
+//    string model_path = "models/resnet34-v1-7.onnx";  // 3x224x224  // okay
+    string model_path = "models/mobilenetv2-7.onnx"; // 3x224x224 // Signal: SIGSEGV (Segmentation fault)
 //    string model_path = "models/vgg16-7.onnx";  // 3xHxW  // okay
 //    string model_path = "models/bvlcalexnet-3.onnx";  // 3x224x224  // The onnx node 'LRN' is not supported yet
-//    string model_path = "models/bvlcalexnet-12.onnx";  // 3x224x224  // The onnx node 'LRN' is not supported yet
-//    string model_path = "models/googlenet-3_simp.onnx";  // 3x224x224  // The onnx node 'LRN' is not supported yet
+//    string model_path = "models/bvlcalexnet-12.onnx";  // 3x224x224  // okay
+//    string model_path = "models/googlenet-3.onnx";  // 3x224x224  // **okay**. bad predictions
 //    string model_path = "models/densenet-3.onnx";  // 3x224x224  // okay
 //    string model_path = "models/inception-v1-3.onnx";  // 3x224x224  // The onnx node 'LRN' is not supported yet
 //    string model_path = "models/efficientnet-lite4-11.onnx";  // 224x224x3  // The onnx node 'LRN' is not supported yet
@@ -98,8 +121,14 @@ int main(int argc, char **argv) {
     int in_channels = 3;
     int in_height = 224;
     int in_width = 224;
+    string channels_order = "rgb";
     vector<int> input_shape = {in_channels, in_height, in_width};
+    vector<float> mean = {0.485, 0.456, 0.406};
+    vector<float> std = {0.229, 0.224, 0.225};
+    bool normalize = true;  // Between [0..1]?
+    bool standarize = true;  // X = (X-mean)/std
 //    vector<int> dimensions_order = {0, 3, 1, 2};
+
 //    // ==========================================================================
 //    input_shape = {input_shape[dimensions_order[1]], input_shape[dimensions_order[2]], input_shape[dimensions_order[3]]};
 
@@ -109,7 +138,7 @@ int main(int argc, char **argv) {
 
     // Import ONNX model
     std::cout << "Importing ONNX..." << std::endl;
-    Net *net = import_net_from_onnx_file(model_path, input_shape);
+    Net *net = import_net_from_onnx_file(model_path, input_shape, 0, LOG_LEVEL::DEBUG);
 
     // ==========================================================================
     // Print and plot our model
@@ -152,7 +181,7 @@ int main(int argc, char **argv) {
     Tensor *image = Tensor::load(image_fname);
 
     // Step 3: Preprocess input. (Look up the preprocessing required at the model's page)
-    Tensor* image_preprocessed = preprocess_input(image, {in_height, in_width});
+    Tensor* image_preprocessed = preprocess_input(image, {in_height, in_width}, mean, std, normalize, standarize, channels_order);
 //    image_preprocessed->permute_(dimensions_order);
 
     // Predict image. Returns a vector of tensors (here one).

examples/onnx/4_onnx_test_gradients.cpp

@@ -28,8 +28,10 @@ int main(int argc, char **argv) {
   bool export_cpu = false;
   bool import_cpu = false;
   for (int i = 1; i < argc; ++i) {
-    if (strcmp(argv[i], "--export-cpu") == 0) export_cpu = true;
-    else if (strcmp(argv[i], "--import-cpu") == 0) import_cpu = true;
+    if (strcmp(argv[i], "--export-cpu") == 0)
+      export_cpu = true;
+    else if (strcmp(argv[i], "--import-cpu") == 0)
+      import_cpu = true;
   }
 
   // Download mnist
@@ -52,9 +54,9 @@ int main(int argc, char **argv) {
   l = ReLu(Conv(l, 32, {3, 3}, {1, 1}));
   l = MaxPool(l, {2, 2});
 
-  l = Reshape(l, {-1});
+  l = Flatten(l);
   l = Dense(l, 128, false);
-  layer out = Activation(Dense(l, num_classes), "softmax");
+  layer out = Softmax(Dense(l, num_classes));
 
   cout << "Creating model" << endl;
   model net = Model({in}, {out});
@@ -63,7 +65,7 @@ int main(int argc, char **argv) {
   // Build model
   cout << "Building the model" << endl;
   build(net,
-        rmsprop(0.01),            // Optimizer
+        adam(0.001),              // Optimizer
         {"soft_cross_entropy"},   // Losses
         {"categorical_accuracy"}, // Metrics
         export_CS,                // Computing service
@@ -112,13 +114,9 @@ int main(int argc, char **argv) {
   // Export trained model
   void *serialized_net_once_trained;
   cout << "Exporting trained weights" << endl;
-  size_t snot_size = serialize_net_to_onnx_pointer(net, serialized_net_once_trained, false);
+  size_t snet_size = serialize_net_to_onnx_pointer(net, serialized_net_once_trained, false);
   cout << "Trained weights exported" << endl;
 
-  // Reset the counter of the layers index
-  LConv::reset_name_counter();
-  LDense::reset_name_counter();
-
   // Import net topology without trained weights
   cout << "Importing original net topology (without training)" << endl;
   Net *imported_net = import_net_from_onnx_pointer(serialized_net, model_size);
@@ -127,7 +125,7 @@ int main(int argc, char **argv) {
   // Build model
   cout << "Building the loaded topology" << endl;
   build(imported_net,
-        rmsprop(0.01),            // Optimizer
+        adam(0.001),              // Optimizer
         {"soft_cross_entropy"},   // Losses
         {"categorical_accuracy"}, // Metrics
         import_CS,                // Computing service
@@ -153,7 +151,7 @@ int main(int argc, char **argv) {
 
   // Set trained weights
   cout << "Putting the trained weights" << endl;
-  set_weights_from_onnx_pointer(imported_net, serialized_net_once_trained, snot_size);
+  set_weights_from_onnx_pointer(imported_net, serialized_net_once_trained, snet_size);
   cout << "Trained weights set" << endl;
 
   // Evaluate with trained weights

examples/onnx/7_onnx_test_gradients_recurrent.cpp

@@ -0,0 +1,164 @@
+/*
+ * EDDL Library - European Distributed Deep Learning Library.
+ * Version: 1.0
+ * copyright (c) 2021, Universitat Politècnica de València (UPV), PRHLT Research
+ * Centre Date: November 2021 Author: PRHLT Research Centre, UPV,
+ * ([email protected]), ([email protected]) All rights reserved
+ */
+
+#include <iostream>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "eddl/apis/eddl.h"
+
+#include "eddl/serialization/onnx/eddl_onnx.h" // Not allowed
+
+using namespace eddl;
+
+///////////////////////////////////////////
+// 7_onnx_test_gradients_recurrent.cpp:
+// An example on how to use the functions
+// for exporting weights and gradients
+// using the ONNX format, with a recurrent
+// network
+///////////////////////////////////////////
+
+int main(int argc, char **argv) {
+  // Read arguments
+  bool export_cpu = false;
+  bool import_cpu = false;
+  for (int i = 1; i < argc; ++i) {
+    if (strcmp(argv[i], "--export-cpu") == 0)
+      export_cpu = true;
+    else if (strcmp(argv[i], "--import-cpu") == 0)
+      import_cpu = true;
+  }
+
+  // Download dataset
+  download_imdb_2000();
+
+  // Settings
+  int epochs = 2;
+  int batch_size = 64;
+  CompServ *export_CS = export_cpu ? CS_CPU() : CS_GPU({1});
+  CompServ *import_CS = import_cpu ? CS_CPU() : CS_GPU({1});
+
+  int length = 250;
+  int embed_dim = 33;
+  int vocsize = 2000;
+
+  // Define network
+  layer in = Input({1}); // 1 word
+  layer l = in;
+
+  layer l_embed = RandomUniform(Embedding(l, vocsize, 1, embed_dim), -0.05, 0.05);
+
+  l = LSTM(l_embed, 37);
+  l = ReLu(Dense(l, 256));
+  layer out = Sigmoid(Dense(l, 1));
+
+  cout << "Creating model" << endl;
+  model net = Model({in}, {out});
+  cout << "Model created" << endl;
+
+  // Build model
+  cout << "Building the model" << endl;
+  build(net,
+        adam(0.001),              // Optimizer
+        {"binary_cross_entropy"}, // Losses
+        {"binary_accuracy"},      // Metrics
+        export_CS,                // Computing service
+        true                      // Enable parameters initialization
+  );
+  cout << "Model is correctly built" << endl;
+
+  cout << "Enabling distributed training" << endl;
+  net->enable_distributed();
+  cout << "Distributed training enabled" << endl;
+
+  // Export the net before training
+  void *serialized_net;
+  cout << "Serializing net (without training) to pointer" << endl;
+  size_t model_size = serialize_net_to_onnx_pointer(net, serialized_net, false);
+  cout << "Net serialized to pointer" << endl;
+
+  // View model
+  summary(net);
+
+  // Load dataset
+  Tensor *x_train = Tensor::load("imdb_2000_trX.bin");
+  Tensor *y_train = Tensor::load("imdb_2000_trY.bin");
+  Tensor *x_test = Tensor::load("imdb_2000_tsX.bin");
+  Tensor *y_test = Tensor::load("imdb_2000_tsY.bin");
+
+  x_train->reshape_({x_train->shape[0], length, 1}); // batch x timesteps x input_dim
+  x_test->reshape_({x_test->shape[0], length, 1}); // batch x timesteps x input_dim
+
+  y_train->reshape_({y_train->shape[0], 1, 1}); // batch x timesteps x input_dim
+  y_test->reshape_({y_test->shape[0], 1, 1});   // batch x timesteps x input_dim
+
+  // Train model
+  cout << "Training the first model" << endl;
+  fit(net, {x_train}, {y_train}, batch_size, epochs);
+
+  // Evaluate
+  cout << "Evaluating the first model" << endl;
+  evaluate(net, {x_test}, {y_test}, batch_size);
+
+  // Export gradients
+  void *serialized_gradients;
+  string path("mnist.onnx");
+  cout << "Exporting gradients" << endl;
+  size_t gradients_size = serialize_net_to_onnx_pointer(net, serialized_gradients, true);
+  cout << "Gradients exported" << endl;
+
+  // Export trained model
+  void *serialized_net_once_trained;
+  cout << "Exporting trained weights" << endl;
+  size_t snet_size = serialize_net_to_onnx_pointer(net, serialized_net_once_trained, false);
+  cout << "Trained weights exported" << endl;
+
+  // Import net topology without trained weights
+  cout << "Importing original net topology (without training)" << endl;
+  Net *imported_net = import_net_from_onnx_pointer(serialized_net, model_size);
+  cout << "Untrained net imported" << endl;
+
+  // Build model
+  cout << "Building the loaded topology" << endl;
+  build(imported_net,
+        adam(0.001),              // Optimizer
+        {"binary_cross_entropy"}, // Losses
+        {"binary_accuracy"},      // Metrics
+        import_CS,                // Computing service
+        false                     // Disable parameters initialization
+  );
+  cout << "Model is correctly built" << endl;
+
+  // View loaded model
+  summary(imported_net);
+
+  // Evaluate with untrained model
+  cout << "Evaluating test with the untrained weights" << endl;
+  evaluate(imported_net, {x_test}, {y_test}, batch_size);
+
+  // Apply grads
+  cout << "Applying grads from training" << endl;
+  apply_grads_from_onnx_pointer(imported_net, serialized_gradients, gradients_size);
+  cout << "Grads applied" << endl;
+
+  // Evaluate net with accumulated gradients applied
+  cout << "Evaluating test after applying gradients" << endl;
+  evaluate(imported_net, {x_test}, {y_test}, batch_size);
+
+  // Set trained weights
+  cout << "Putting the trained weights" << endl;
+  set_weights_from_onnx_pointer(imported_net, serialized_net_once_trained, snet_size);
+  cout << "Trained weights set" << endl;
+
+  // Evaluate with trained weights
+  cout << "Evaluating test after putting the trained weights" << endl;
+  evaluate(imported_net, {x_test}, {y_test}, batch_size);
+
+  return 0;
+}

include/eddl/layers/core/layer_core.h

@@ -88,6 +88,7 @@ class LEmbedding : public LinLayer {
     bool mask_zeros;
     Tensor *E;
     Tensor *gE;
+    Tensor *acc_gE;
     vector<int> sind;
     static int total_layers;
 
@@ -103,6 +104,16 @@ class LEmbedding : public LinLayer {
 
     void backward() override;
 
+    void update_weights(vector<Tensor*> weights) override;
+
+    void accumulate_accumulated_gradients(vector<Tensor*> grads) override;
+
+    void reset_accumulated_gradients() override;
+
+    void apply_accumulated_gradients() override;
+
+    void enable_distributed() override;
+
     string plot(int c) override;
 
 };

include/eddl/layers/layer.h

@@ -197,10 +197,19 @@ class MLayer : public Layer {
 
     void backward() override {}
 
+    void update_weights(vector<Tensor*> weights) override {}
+
+    void accumulate_accumulated_gradients(vector<Tensor*> grads) override {}
+
+    void reset_accumulated_gradients() override {}
+
+    void apply_accumulated_gradients() override {}
+
     Layer *share(int c, int bs, vector<Layer *> p) override { return nullptr; }
 
     Layer *clone(int c, int bs, vector<Layer *> p, int todev) override { return nullptr; }
 
+    void enable_distributed() override {};
 };
 
 #endif //EDDL_LAYER_H