csv client lib (IntelLabs#78)

* Format CMakeLists.txt

* Add CSV CPP Client Plugin

* Fix the VDMS path dependencies in csv_plugin library

* Add the multithread layer class in CSVParser class that uses the utilities in CSVPArserUtil.cpp and add the test in tests folder

* merge csv plugin in cleint folder

* Update check-in Dockerfiles to include lib

* Add the csv-plugin as CSVParser in the VDMS::Client CPP folder as a utility

* Remove COpy csv-cpp-lib

* fix the failing tests

* adjust run python_tets

* fix the stash conflict

* wprking on csv tests

* Working os csv tests

* Update run_coverage_cpp.sh

Include .cpp files

* Update run_coverage_cpp.sh

Add flag to overcome known error (https://gcovr.com/en/master/guide/gcov_parser.html#negative-hit-counts)

* stash changes

* Configue the prot of csv parser

* Trying to add the video tesing in csv

* Fi the test script

* fix run_python_test

* Uncoment the Update in CSV

* Csv client lib (IntelLabs#95)

* Update python coverage test to exclude tests folder

* Remove unused old folder (csv-cpp-lib), remove hardcoded ragaad paths from CMakeLists files, remove commented code

* Fix splitrow, add bin to supported image format, uncomment constraints for images, use CSVformat100.csv for entity test, add operations to image and video csv

* Update tests/unit_tests/client_add_entity.cc

Removed empty line (only change to file)

* Update CMakeLists.txt

Remove empty line (only change to file)

* Update tests/main.cc

remove empty line

* Csv client lib (IntelLabs#96)

* Fix ops in video csv, update csvs for additional testing, improve isInt

* Add additional tests and catch case for unkonwn commandtype

* Update tests/main.cc

---------

Co-authored-by: Lacewell, Chaunte W <[email protected]>

CMakeLists.txt

@@ -40,30 +40,30 @@ else()
 
   link_directories(/usr/local/lib /usr/lib/x86_64-linux-gnu/)
   include_directories(/usr/include/jsoncpp utils/include/ src/pmgd/include src/pmgd/util include/ src/vcl /usr/include ${CMAKE_CURRENT_BINARY_DIR}/utils/src/protobuf)
-  add_library(dms SHARED  
-                        src/BoundingBoxCommand.cc 
-                        src/BlobCommand.cc 
-                        src/CommunicationManager.cc 
-                        src/DescriptorsCommand.cc 
-                        src/DescriptorsManager.cc 
-                        src/ExceptionsCommand.cc 
-                        src/ImageCommand.cc 
-                        src/PMGDIterators.cc 
-                        src/PMGDQuery.cc 
-                        src/PMGDQueryHandler.cc 
-                        src/QueryHandler.cc 
-                        src/QueryMessage.cc 
-                        src/RSCommand.cc 
-                        src/SearchExpression.cc 
-                        src/Server.cc 
-                        src/VDMSConfig.cc 
-                        src/VideoCommand.cc 
-                        src/AutoDeleteNode.cc 
+  add_library(dms SHARED
+                        src/BoundingBoxCommand.cc
+                        src/BlobCommand.cc
+                        src/CommunicationManager.cc
+                        src/DescriptorsCommand.cc
+                        src/DescriptorsManager.cc
+                        src/ExceptionsCommand.cc
+                        src/ImageCommand.cc
+                        src/PMGDIterators.cc
+                        src/PMGDQuery.cc
+                        src/PMGDQueryHandler.cc
+                        src/QueryHandler.cc
+                        src/QueryMessage.cc
+                        src/RSCommand.cc
+                        src/SearchExpression.cc
+                        src/Server.cc
+                        src/VDMSConfig.cc
+                        src/VideoCommand.cc
+                        src/AutoDeleteNode.cc
                         ${PROTO_SRCS} ${PROTO_HDRS}
                         )
   target_link_libraries(dms vcl pmgd pmgd-util protobuf vdms-utils pthread)
 
 
   add_executable(vdms src/vdms.cc)
   target_link_libraries(vdms dms vdms_protobuf vcl tiledb faiss flinng jsoncpp ${OpenCV_LIBS})
-endif ()  
+endif ()

client/cpp/BoundingBoxQueryParser.h

@@ -0,0 +1,109 @@
+#include "CSVParserUtil.h"
+namespace VDMS {
+class BoundingBoxQueryParser : public CSVParserUtil{
+    private:
+        vector<string> rectangleKeys{"x","y","w","h"};
+        void parseRectangle(string row,string queryType,Json::Value &aquery);
+    public:
+        VDMS::Response ParseAddBoundingBox(vector<string> row, vector<string> cols);
+        // VDMS::Response ParseUpdateBoundingBox(vector<string> row, vector<string>& cols);
+
+};
+};
+
+VDMS::Response VDMS::BoundingBoxQueryParser::ParseAddBoundingBox(vector<string> row, vector<string> columnNames){
+    if (row.empty() || row[0].empty()) {
+        throw "please provide rectangle details";
+    }
+
+    Json::Value aquery;
+    Json::Value allquery;
+    std::string command_name = "AddBoundingBox";
+
+    aquery["AddBoundingBox"]["_ref"] = 1;
+    // aquery["AddBoundingBox"]["image"] = 3;
+
+    Json::Value aqueryf;
+    // aqueryf["FindImage"]["_ref"] = 3;
+
+    parseRectangle(row[0], "AddBoundingBox", aquery);
+
+    for (int j = 1; j < columnNames.size(); j++){
+        const string& columnName = columnNames[j];
+        const string& cellValue = row[j];
+
+        if (cellValue.empty()) {
+            continue;
+        }
+
+        if (columnName.find("prop_") != string::npos){
+            parseProperty(columnName, cellValue, command_name, aquery);
+        }
+        // else if (columnName.find("cons_") != string::npos){
+        //     std::string find_image = "FindImage";
+        //     parseConstraints(columnName, cellValue, find_image, aqueryf);
+        // }
+    }
+
+    // allquery.append(aqueryf);
+
+    allquery.append(aquery);
+   return  send_to_vdms(allquery);
+}
+
+
+void VDMS::BoundingBoxQueryParser::parseRectangle(string row, string queryType, Json::Value& aquery) {
+    Json::Value rec;
+    string::size_type start = 0;
+    for (int c = 0; c < 4; c++) {
+        string::size_type end = row.find(',', start);
+        if (end == string::npos && c < 3) {
+            throw "rectangle data should have four values";
+        }
+        string substr = row.substr(start, end - start);
+         try {
+            int intVal = stoi(substr);
+            rec[rectangleKeys[c]] = intVal;
+        } catch (const invalid_argument&) {
+            try {
+                float floatVal = stof(substr);
+                rec[rectangleKeys[c]] = floatVal;
+            } catch (const invalid_argument&) {
+                throw "invalid datatype of the rectangle data";
+            }
+        }
+
+        start = end + 1;
+    }
+    aquery[queryType]["rectangle"] = rec;
+}
+
+// VDMS::Response VDMS::BoundingBoxQueryParser::ParseUpdateBoundingBox(vector<string> row, vector<string>& columnNames){
+//     Json::Value aquery;
+//     Json::Value allquery;
+//     aquery["UpdateBoundingBox"]["_ref"]=3;
+//     std::string command_name="UpdateBoundingBox";
+//     if(row[0]!=""){
+//         parseRectangle(row[0],command_name,aquery);    
+//     }
+//     for(int j=1;j<columnNames.size();j++){
+//         if(row[j]!=""){
+//             if(columnNames[j].find("prop_")!=string::npos){
+//                 parseProperty(columnNames[j],row[j],command_name,aquery);
+//             }
+//             else if(columnNames[j].find("cons_")!=string::npos){
+//                parseConstraints(columnNames[j],row[j],command_name,aquery);
+//             }
+//             else if(columnNames[j].find("remv_")!=string::npos){
+//                 vector<string> rowvec;
+//                 splitRowOnComma(row[j],rowvec);
+//                 for(int v=0;v<rowvec.size();v++){
+//                     aquery["UpdateBoundingBox"]["remove_props"].append(rowvec[v]);
+//                 }
+//             }
+//         }
+//     }
+//     allquery.append(aquery);
+//     return send_to_vdms(allquery);
+// }
+

client/cpp/CMakeLists.txt

@@ -1,8 +1,13 @@
 cmake_minimum_required (VERSION 3.10)
 project(vdms_client)
 find_package( OpenCV REQUIRED )
+find_package( Threads REQUIRED )
+include_directories(../../utils/include/ )
 
-include_directories(../../utils/include/)
-add_library(vdms-client SHARED VDMSClient.cc)
-target_link_libraries(vdms-client protobuf vdms_protobuf vdms-utils)
+
+add_library(  vdms-client SHARED 
+                VDMSClient.cc
+                CSVParserUtil.cpp
+)
+target_link_libraries(vdms-client protobuf vdms_protobuf vdms-utils pthread ${CMAKE_THREAD_LIBS_INIT} )
 

client/cpp/CSVParser.h

@@ -0,0 +1,73 @@
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <string>
+#include <thread>
+#include <mutex>
+#include "rapidcsv.h"
+#include "CSVParserUtil.h"
+
+namespace VDMS {
+class CSVParser {
+public:
+    CSVParser(std::string filename, size_t num_threads, std::string server, int port) : m_filename(filename), m_num_threads(num_threads),vdms_server(server),vdms_port(port) {}
+    std::vector<VDMS::Response> parse_csv_lines(const std::string& filename, int start_line, int end_line, std::mutex& mutex, std::vector<VDMS::Response>& all_results, const size_t thread_id)
+    { 
+    rapidcsv::Document csv(filename);
+
+    size_t rowCount = csv.GetRowCount();
+    std::vector<std::string> columnNames = csv.GetColumnNames();
+    VDMS::CSVParserUtil csv_util(vdms_server, vdms_port, columnNames, static_cast<int>(thread_id));
+    for (int i = start_line; i < end_line; ++i)
+    {
+        std::vector<std::string> row = csv.GetRow<std::string>(i);
+        VDMS::Response result = csv_util.parse_row(row);
+
+        std::lock_guard<std::mutex> lock(mutex);
+        all_results.push_back(result);
+    }
+
+    return all_results;
+  }
+std::vector <VDMS::Response> parse() {
+        auto start = std::chrono::high_resolution_clock::now();
+
+        std::ifstream file(m_filename);
+        if (!file) {
+            std::cerr << "Error opening file\n";
+
+        }
+
+        int num_lines = std::count(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), '\n');
+        std::size_t lines_per_thread = num_lines / m_num_threads;
+
+        std::mutex mutex;
+        std::vector<std::thread> threads;
+        std::vector <VDMS::Response> all_results;
+
+        for (size_t i = 0; i < m_num_threads; i++) {
+            size_t start_line = i * lines_per_thread;
+            size_t end_line = (i == m_num_threads - 1) ? num_lines-1 : (i + 1) * lines_per_thread;
+
+            threads.emplace_back(&CSVParser::parse_csv_lines, this, std::ref(m_filename), start_line, end_line, std::ref(mutex),std::ref(all_results), i);
+        }
+
+        for (auto& thread : threads) {
+            thread.join();
+        }
+
+        auto finish = std::chrono::high_resolution_clock::now();
+        std::chrono::duration<double> elapsed = finish - start;
+        std::cout << "Elapsed time: " << elapsed.count() << " s\n";
+        return all_results;
+    }
+
+private:
+    std::string m_filename;
+    size_t m_num_threads;
+    std::string vdms_server;
+    int vdms_port;
+
+
+ };
+};