From beda7a5a46394b66e4c7c9ff061839417594ac27 Mon Sep 17 00:00:00 2001 From: Ragaad Date: Fri, 24 Mar 2023 12:42:54 -0700 Subject: [PATCH] csv client lib (#78) * Format CMakeLists.txt * Add CSV CPP Client Plugin * Fix the VDMS path dependencies in csv_plugin library * Add the multithread layer class in CSVParser class that uses the utilities in CSVPArserUtil.cpp and add the test in tests folder * merge csv plugin in cleint folder * Update check-in Dockerfiles to include lib * Add the csv-plugin as CSVParser in the VDMS::Client CPP folder as a utility * Remove COpy csv-cpp-lib * fix the failing tests * adjust run python_tets * fix the stash conflict * wprking on csv tests * Working os csv tests * Update run_coverage_cpp.sh Include .cpp files * Update run_coverage_cpp.sh Add flag to overcome known error (https://gcovr.com/en/master/guide/gcov_parser.html#negative-hit-counts) * stash changes * Configue the prot of csv parser * Trying to add the video tesing in csv * Fi the test script * fix run_python_test * Uncoment the Update in CSV * Csv client lib (#95) * Update python coverage test to exclude tests folder * Remove unused old folder (csv-cpp-lib), remove hardcoded ragaad paths from CMakeLists files, remove commented code * Fix splitrow, add bin to supported image format, uncomment constraints for images, use CSVformat100.csv for entity test, add operations to image and video csv * Update tests/unit_tests/client_add_entity.cc Removed empty line (only change to file) * Update CMakeLists.txt Remove empty line (only change to file) * Update tests/main.cc remove empty line * Csv client lib (#96) * Fix ops in video csv, update csvs for additional testing, improve isInt * Add additional tests and catch case for unkonwn commandtype * Update tests/main.cc --------- Co-authored-by: Lacewell, Chaunte W --- CMakeLists.txt | 40 +- client/cpp/BoundingBoxQueryParser.h | 109 ++ client/cpp/CMakeLists.txt | 11 +- client/cpp/CSVParser.h | 73 ++ client/cpp/CSVParserUtil.cpp | 652 ++++++++++ client/cpp/CSVParserUtil.h | 102 ++ client/cpp/ConnectionQueryParser.h | 115 ++ client/cpp/DescriptorQueryParser.h | 56 + client/cpp/DescriptorSetQueryParser.h | 55 + client/cpp/EntityQueryParser.h | 92 ++ client/cpp/ImageQueryParser.h | 103 ++ client/cpp/VDMSClient.cc | 9 + client/cpp/VDMSClient.h | 8 +- client/cpp/VideoQueryParser.h | 84 ++ client/cpp/rapidcsv.h | 1720 +++++++++++++++++++++++++ docker/check-in/run_coverage_cpp.sh | 6 +- tests/CMakeLists.txt | 1 + tests/csv_samples/CSVformat100.csv | 96 ++ tests/csv_samples/Descriptor.csv | 6 + tests/csv_samples/DescriptorSet.csv | 7 + tests/csv_samples/Image.csv | 11 + tests/csv_samples/Rectangle.csv | 13 + tests/csv_samples/Video.csv | 6 + tests/csv_samples/blob_1.txt | 1 + tests/csv_samples/connection.csv | 5 + tests/csv_samples/person.csv | 6 + tests/run_tests.sh | 3 +- tests/unit_tests/client_blob.cc | 13 +- tests/unit_tests/client_csv.cc | 238 ++++ tests/unit_tests/client_videos.cc | 2 +- tests/unit_tests/meta_data_helper.h | 7 +- 31 files changed, 3612 insertions(+), 38 deletions(-) create mode 100644 client/cpp/BoundingBoxQueryParser.h create mode 100644 client/cpp/CSVParser.h create mode 100644 client/cpp/CSVParserUtil.cpp create mode 100644 client/cpp/CSVParserUtil.h create mode 100644 client/cpp/ConnectionQueryParser.h create mode 100644 client/cpp/DescriptorQueryParser.h create mode 100644 client/cpp/DescriptorSetQueryParser.h create mode 100644 client/cpp/EntityQueryParser.h create mode 100644 client/cpp/ImageQueryParser.h create mode 100644 client/cpp/VideoQueryParser.h create mode 100644 client/cpp/rapidcsv.h create mode 100644 tests/csv_samples/CSVformat100.csv create mode 100644 tests/csv_samples/Descriptor.csv create mode 100644 tests/csv_samples/DescriptorSet.csv create mode 100644 tests/csv_samples/Image.csv create mode 100644 tests/csv_samples/Rectangle.csv create mode 100644 tests/csv_samples/Video.csv create mode 100644 tests/csv_samples/blob_1.txt create mode 100644 tests/csv_samples/connection.csv create mode 100644 tests/csv_samples/person.csv create mode 100644 tests/unit_tests/client_csv.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index b51e6fbd..afda38bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,25 +40,25 @@ else() link_directories(/usr/local/lib /usr/lib/x86_64-linux-gnu/) include_directories(/usr/include/jsoncpp utils/include/ src/pmgd/include src/pmgd/util include/ src/vcl /usr/include ${CMAKE_CURRENT_BINARY_DIR}/utils/src/protobuf) - add_library(dms SHARED - src/BoundingBoxCommand.cc - src/BlobCommand.cc - src/CommunicationManager.cc - src/DescriptorsCommand.cc - src/DescriptorsManager.cc - src/ExceptionsCommand.cc - src/ImageCommand.cc - src/PMGDIterators.cc - src/PMGDQuery.cc - src/PMGDQueryHandler.cc - src/QueryHandler.cc - src/QueryMessage.cc - src/RSCommand.cc - src/SearchExpression.cc - src/Server.cc - src/VDMSConfig.cc - src/VideoCommand.cc - src/AutoDeleteNode.cc + add_library(dms SHARED + src/BoundingBoxCommand.cc + src/BlobCommand.cc + src/CommunicationManager.cc + src/DescriptorsCommand.cc + src/DescriptorsManager.cc + src/ExceptionsCommand.cc + src/ImageCommand.cc + src/PMGDIterators.cc + src/PMGDQuery.cc + src/PMGDQueryHandler.cc + src/QueryHandler.cc + src/QueryMessage.cc + src/RSCommand.cc + src/SearchExpression.cc + src/Server.cc + src/VDMSConfig.cc + src/VideoCommand.cc + src/AutoDeleteNode.cc ${PROTO_SRCS} ${PROTO_HDRS} ) target_link_libraries(dms vcl pmgd pmgd-util protobuf vdms-utils pthread) @@ -66,4 +66,4 @@ else() add_executable(vdms src/vdms.cc) target_link_libraries(vdms dms vdms_protobuf vcl tiledb faiss flinng jsoncpp ${OpenCV_LIBS}) -endif () +endif () diff --git a/client/cpp/BoundingBoxQueryParser.h b/client/cpp/BoundingBoxQueryParser.h new file mode 100644 index 00000000..63c3e7d0 --- /dev/null +++ b/client/cpp/BoundingBoxQueryParser.h @@ -0,0 +1,109 @@ +#include "CSVParserUtil.h" +namespace VDMS { +class BoundingBoxQueryParser : public CSVParserUtil{ + private: + vector rectangleKeys{"x","y","w","h"}; + void parseRectangle(string row,string queryType,Json::Value &aquery); + public: + VDMS::Response ParseAddBoundingBox(vector row, vector cols); + // VDMS::Response ParseUpdateBoundingBox(vector row, vector& cols); + +}; +}; + +VDMS::Response VDMS::BoundingBoxQueryParser::ParseAddBoundingBox(vector row, vector columnNames){ + if (row.empty() || row[0].empty()) { + throw "please provide rectangle details"; + } + + Json::Value aquery; + Json::Value allquery; + std::string command_name = "AddBoundingBox"; + + aquery["AddBoundingBox"]["_ref"] = 1; + // aquery["AddBoundingBox"]["image"] = 3; + + Json::Value aqueryf; + // aqueryf["FindImage"]["_ref"] = 3; + + parseRectangle(row[0], "AddBoundingBox", aquery); + + for (int j = 1; j < columnNames.size(); j++){ + const string& columnName = columnNames[j]; + const string& cellValue = row[j]; + + if (cellValue.empty()) { + continue; + } + + if (columnName.find("prop_") != string::npos){ + parseProperty(columnName, cellValue, command_name, aquery); + } + // else if (columnName.find("cons_") != string::npos){ + // std::string find_image = "FindImage"; + // parseConstraints(columnName, cellValue, find_image, aqueryf); + // } + } + + // allquery.append(aqueryf); + + allquery.append(aquery); + return send_to_vdms(allquery); +} + + +void VDMS::BoundingBoxQueryParser::parseRectangle(string row, string queryType, Json::Value& aquery) { + Json::Value rec; + string::size_type start = 0; + for (int c = 0; c < 4; c++) { + string::size_type end = row.find(',', start); + if (end == string::npos && c < 3) { + throw "rectangle data should have four values"; + } + string substr = row.substr(start, end - start); + try { + int intVal = stoi(substr); + rec[rectangleKeys[c]] = intVal; + } catch (const invalid_argument&) { + try { + float floatVal = stof(substr); + rec[rectangleKeys[c]] = floatVal; + } catch (const invalid_argument&) { + throw "invalid datatype of the rectangle data"; + } + } + + start = end + 1; + } + aquery[queryType]["rectangle"] = rec; +} + +// VDMS::Response VDMS::BoundingBoxQueryParser::ParseUpdateBoundingBox(vector row, vector& columnNames){ +// Json::Value aquery; +// Json::Value allquery; +// aquery["UpdateBoundingBox"]["_ref"]=3; +// std::string command_name="UpdateBoundingBox"; +// if(row[0]!=""){ +// parseRectangle(row[0],command_name,aquery); +// } +// for(int j=1;j rowvec; +// splitRowOnComma(row[j],rowvec); +// for(int v=0;v +#include +#include +#include +#include +#include +#include "rapidcsv.h" +#include "CSVParserUtil.h" + +namespace VDMS { +class CSVParser { +public: + CSVParser(std::string filename, size_t num_threads, std::string server, int port) : m_filename(filename), m_num_threads(num_threads),vdms_server(server),vdms_port(port) {} + std::vector parse_csv_lines(const std::string& filename, int start_line, int end_line, std::mutex& mutex, std::vector& all_results, const size_t thread_id) + { + rapidcsv::Document csv(filename); + + size_t rowCount = csv.GetRowCount(); + std::vector columnNames = csv.GetColumnNames(); + VDMS::CSVParserUtil csv_util(vdms_server, vdms_port, columnNames, static_cast(thread_id)); + for (int i = start_line; i < end_line; ++i) + { + std::vector row = csv.GetRow(i); + VDMS::Response result = csv_util.parse_row(row); + + std::lock_guard lock(mutex); + all_results.push_back(result); + } + + return all_results; + } +std::vector parse() { + auto start = std::chrono::high_resolution_clock::now(); + + std::ifstream file(m_filename); + if (!file) { + std::cerr << "Error opening file\n"; + + } + + int num_lines = std::count(std::istreambuf_iterator(file), std::istreambuf_iterator(), '\n'); + std::size_t lines_per_thread = num_lines / m_num_threads; + + std::mutex mutex; + std::vector threads; + std::vector all_results; + + for (size_t i = 0; i < m_num_threads; i++) { + size_t start_line = i * lines_per_thread; + size_t end_line = (i == m_num_threads - 1) ? num_lines-1 : (i + 1) * lines_per_thread; + + threads.emplace_back(&CSVParser::parse_csv_lines, this, std::ref(m_filename), start_line, end_line, std::ref(mutex),std::ref(all_results), i); + } + + for (auto& thread : threads) { + thread.join(); + } + + auto finish = std::chrono::high_resolution_clock::now(); + std::chrono::duration elapsed = finish - start; + std::cout << "Elapsed time: " << elapsed.count() << " s\n"; + return all_results; + } + +private: + std::string m_filename; + size_t m_num_threads; + std::string vdms_server; + int vdms_port; + + + }; +}; \ No newline at end of file diff --git a/client/cpp/CSVParserUtil.cpp b/client/cpp/CSVParserUtil.cpp new file mode 100644 index 00000000..207f1c92 --- /dev/null +++ b/client/cpp/CSVParserUtil.cpp @@ -0,0 +1,652 @@ + +#include +#include +#include "CSVParserUtil.h" +#include "rapidcsv.h" +#include "EntityQueryParser.h" +#include "ImageQueryParser.h" +#include "VideoQueryParser.h" +#include "DescriptorQueryParser.h" +#include "DescriptorSetQueryParser.h" +#include "BoundingBoxQueryParser.h" +#include "ConnectionQueryParser.h" +#include +static std::mutex barrier; +std::mutex mtx; +using namespace std; +using namespace std::chrono; +using namespace VDMS; + +VDMS::CSVParserUtil::CSVParserUtil() : vdms_server("localhost"), vdms_port(55558), + vdms_client(std::unique_ptr(new VDMS::VDMSClient(vdms_server, vdms_port))) +{ + initCommandsMap(); +} + +VDMS::CSVParserUtil::CSVParserUtil(const std::string &server, int port, const std::vector columnNames, int index) : vdms_server(server), + vdms_port(port), + _columnNames(columnNames), + id(index), vdms_client(std::unique_ptr(new VDMS::VDMSClient(vdms_server, vdms_port))) +{ + initCommandsMap(); +} +void VDMS::CSVParserUtil::initCommandsMap() +{ + commands = { + {"EntityClass", EntityClass}, + {"ConnectionClass", ConnectionClass}, + {"ImagePath", ImagePath}, + {"VideoPath", VideoPath}, + {"DescriptorType", DescriptorType}, + {"DescriptorClass", DescriptorClass}, + {"RectangleBound", RectangleBound}, + {"EntityUpdate", EntityUpdate}, + {"ConnectionUpdate", ConnectionUpdate}, + {"ImageUpdate", ImageUpdate}, + {"RectangleUpdate", RectangleUpdate}}; +} +string VDMS::CSVParserUtil::function_accessing_columnNames(int i) +{ + std::lock_guard lock(mtx); + + return _columnNames[i]; +} +VDMS::Response VDMS::CSVParserUtil::parse_row(std::vector &row) +{ + VDMS::Response result; + + VDMS::CSVParserUtil::commandType queryType = get_query_type(_columnNames[0]); + switch (queryType) + { + case commandType::AddEntity: + { + EntityQueryParser entityquery; + result = entityquery.ParseAddEntity(row, _columnNames); + } + + break; + case commandType::AddConnection: + { + + ConnectionQueryParser connectionquery; + result = connectionquery.ParseAddConnection(row, _columnNames); + } + break; + + case commandType::AddImage: + { + ImageQueryParser imagequery; + result = imagequery.ParseAddImage(row, _columnNames); + } + break; + case commandType::AddVideo: + { + VideoQueryParser videoquery; + result = videoquery.ParseAddVideo(row, _columnNames); + } + break; + case commandType::AddDescriptorSet: + { + DescriptorSetQueryParser descriptorsetquery; + + result = descriptorsetquery.ParseAddDescriptorSet(row, _columnNames); + } + break; + case commandType::AddDescriptor: + { + DescriptorQueryParser descriptorquery; + result = descriptorquery.ParseAddDescriptor(row, _columnNames, id); + } + break; + case commandType::AddBoundingBox: + { + BoundingBoxQueryParser boundingboxquery; + result = boundingboxquery.ParseAddBoundingBox(row, _columnNames); + } + break; + // case commandType::UpdateEntity:{ + // EntityQueryParser update_entityquery; + // update_entityquery.ParseUpdateEntity(row, _columnNames); + // } + // break; + // case commandType::UpdateConnection:{ + // ConnectionQueryParser update_connectionquery; + // update_connectionquery.ParseUpdateConnection(row,allquery,i+1,_columnNames); + // } + // break; + // case commandType::UpdateImage:{ + // ImageQueryParser update_image_query; + // update_image_query.ParseUpdateImage(row,allquery,i+1,_columnNames); + // } + // break; + // case commandType::UpdateBoundingBox:{ + // BoundingBoxQueryParser update_boundingboxquery; + // update_boundingboxquery.ParseUpdateBoundingBox(row,allquery,i+1,_columnNames); + // } + // break; + case commandType::UNKNOWN:{ + Json::Value results; + results["status"] = -1; + results["info"] = "Command does not exist"; + result.json = results.toStyledString(); + } + break; + } + return result; +} + +int VDMS::CSVParserUtil::isBool(const std::string &s) +{ + std::string lower = s; + std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); + if (lower == "true") + return 1; + else if (lower == "false") + return 2; + return 0; +} + +bool VDMS::CSVParserUtil::isFloat(const std::string &s) +{ + std::istringstream ss(s); + float x; + char c; + return (ss >> x) && !(ss >> c); +} + +bool VDMS::CSVParserUtil::isInt(const std::string &s) +{ + std::istringstream ss(s); + int x; + char c; + return (ss >> x) && (floor(x)) && !(ss >> c); +} + +VDMS::CSVParserUtil::commandType VDMS::CSVParserUtil::get_query_type(const string &str) +{ + CSVParserUtil::commandType querytype; + + std::lock_guard lock(CSVParserUtil::querytype_mutex); + std::map::iterator iter; + iter = commands.find(str); + if (iter == commands.end()) { + return commandType::UNKNOWN; + } else { + switch (commands[str]) + { + case EntityClass: + querytype = commandType::AddEntity; + break; + + case ConnectionClass: + querytype = commandType::AddConnection; + break; + case ImagePath: + querytype = commandType::AddImage; + break; + case VideoPath: + querytype = commandType::AddVideo; + break; + case DescriptorType: + querytype = commandType::AddDescriptorSet; + break; + case DescriptorClass: + querytype = commandType::AddDescriptor; + break; + case RectangleBound: + querytype = commandType::AddBoundingBox; + + break; + // case EntityUpdate: + // querytype = commandType::UpdateEntity; + + // break; + // case ConnectionUpdate: + // querytype = commandType::UpdateConnection; + + // break; + // case ImageUpdate: + // querytype = commandType::UpdateImage; + // break; + // case RectangleUpdate: + // querytype = commandType::UpdateBoundingBox; + // break; + } + + // std::cout << " I executed queryType "<< querytype << std::endl; + return querytype; + } +} + +VDMS::CSVParserUtil::DATATYPE VDMS::CSVParserUtil::getDataType(const string &str, const string &propname) +{ + if (propname.substr(0, 5) == "date:") + return DATATYPE::DATE; + else if (isInt(str)) + return DATATYPE::INTEGER; + else if (isFloat(str)) + return DATATYPE::FLOAT; + else if (isBool(str) == 1) + return DATATYPE::TRUE; + else if (isBool(str) == 2) + return DATATYPE::FALSE; + else + return DATATYPE::STRING; +} + +void VDMS::CSVParserUtil::parseProperty(const string &columnNames, const string &row, const string &queryType, Json::Value &aquery) +{ + std::lock_guard lock(CSVParserUtil::aquery_mutex); + string propname = columnNames.substr(5, string::npos); + // std::cout << "Inside parseProp " << propname < lock(CSVParserUtil::cons_mutex); + vector consvals = spiltrow(row); + string consname = consvals[0]; + if (consname.substr(0, 5) == "date:") + { + for (int z = 1; z < consvals.size(); z++) + { + if (z % 2 == 1) + { + aquery[queryType]["constraints"][consname.substr(5, string::npos)].append(consvals[z]); + } + else + { + Json::Value date; + date["_date"] = consvals[z]; + aquery[queryType]["constraints"][consname.substr(5, string::npos)].append(date); + } + } + } + else + { + for (int z = 1; z < consvals.size(); z++) + { + CSVParserUtil::DATATYPE dtype = getDataType(consvals[z], consname); + if (dtype == DATATYPE::TRUE) + { + aquery[queryType]["constraints"][consname].append(true); + } + else if (dtype == DATATYPE::FALSE) + { + aquery[queryType]["constraints"][consname].append(false); + } + else if (dtype == DATATYPE::INTEGER) + { + aquery[queryType]["constraints"][consname].append(stoi(consvals[z])); + } + else if (dtype == DATATYPE::FLOAT) + { + aquery[queryType]["constraints"][consname].append(stof(consvals[z])); + } + else + { + aquery[queryType]["constraints"][consname].append(consvals[z]); + } + } + } +} + +vector VDMS::CSVParserUtil::spiltrow(const string &str) +{ + string row = str; + vector rowv; + int start = 0; + for (int i = 0; i < row.size(); i++) + { + if ((row[i] == '<') || (row[i] == '>') || (row[i] == '=')) + { + if (row[i + 1] == '=') + { + rowv.push_back(row.substr(start, i - start)); + rowv.push_back(row.substr(i, 2)); + i++; + } + else + { + rowv.push_back(row.substr(start, i - start)); + rowv.push_back(row.substr(i, 1)); + } + start = i + 1; + } + else if (row[i] == ',') + { + row.erase(i, 1); + i--; + } + } + rowv.push_back(row.substr(start, string::npos)); + return rowv; +} +void VDMS::CSVParserUtil::parseOperations(string columnNames, string row, string queryType, Json::Value &aquery) +{ + + std::lock_guard lock(CSVParserUtil::ops_mutex); + string type = columnNames.substr(4, string::npos); + Json::Value opsjson; + vector opsKeys; + if (isValidOpsType(type)) + opsjson["type"] = type; + else + throw "invalid operation command name"; + vector rowvec; + int c; + splitRowOnComma(row, rowvec); + if (type == "crop") + { + if (rowvec.size() <= 3 || rowvec.size() > 4) + throw "For crop data should be of size 4"; + opsKeys = {"x", "y", "width", "height"}; + for (c = 0; c < rowvec.size(); c++) + { + string substr = rowvec[c]; + DATATYPE dType = isValidDataType(substr, 2); + + if (dType == DATATYPE::INTEGER) + opsjson[opsKeys[c]] = stoi(substr); + else if (dType == DATATYPE::FLOAT) + opsjson[opsKeys[c]] = stof(substr); + + else + { + throw "Numeric data is required for crop command"; + } + } + } + + else if (type == "threshold") + { + DATATYPE dType = isValidDataType(row, 2); + if (dType == DATATYPE::INTEGER) + opsjson["value"] = stoi(row); + else if (dType == DATATYPE::FLOAT) + opsjson["value"] = stof(row); + + else + { + throw "Numeric data is required for resize command"; + } + } + + else if (type == "resize") + { + if (rowvec.size() <= 1 || rowvec.size() > 2) + throw "For resize data should be of size 2"; + opsKeys = {"width", "height"}; + for (c = 0; c < rowvec.size(); c++) + { + string substr = rowvec[c]; + DATATYPE dType = isValidDataType(substr, 2); + + if (dType == DATATYPE::INTEGER) + opsjson[opsKeys[c]] = stoi(substr); + else if (dType == DATATYPE::FLOAT) + opsjson[opsKeys[c]] = stof(substr); + + else + { + throw "Numeric data is required for resize command"; + } + } + } + + else if (type == "flip") + { + DATATYPE dType = isValidDataType(row, 2); + if (dType == DATATYPE::INTEGER) + { + opsjson["code"] = stoi(row); + } + else + { + throw "Numeric data is required for flip command"; + } + } + + else if (type == "rotate") + { + if (rowvec.size() <= 1 || rowvec.size() > 2) + throw "For rotate data should be of size 2"; + opsKeys = {"angle", "resize"}; + for (c = 0; c < rowvec.size(); c++) + { + string substr = rowvec[c]; + if (c == 0) + { + DATATYPE dType = isValidDataType(substr, 2); + if (dType == DATATYPE::INTEGER) + opsjson[opsKeys[c]] = stoi(substr); + else if (dType == DATATYPE::FLOAT) + opsjson[opsKeys[c]] = stof(substr); + + else + { + throw "Numeric data is required for resize command"; + } + } + else if (c == 1) + { + DATATYPE dType = isValidDataType(substr, 1); + if (dType == DATATYPE::TRUE) + opsjson[opsKeys[c]] = true; + else if (dType == DATATYPE::FALSE) + opsjson[opsKeys[c]] = false; + + else + { + throw "Boolean data is required for rotate resize"; + } + } + } + } + + else if (type == "interval") + { + if (rowvec.size() <= 2 || rowvec.size() > 3) + throw "interval operation has 3 values to specify"; + opsKeys = {"start", "stop", "step"}; + for (c = 0; c < rowvec.size(); c++) + { + string substr = rowvec[c]; + DATATYPE dType = isValidDataType(substr, 2); + if (dType == DATATYPE::INTEGER) + { + opsjson[opsKeys[c]] = stoi(substr); + } + else + { + throw "Numeric datatype is required for the interval"; + } + } + } + + aquery[queryType]["operations"].append(opsjson); +} + +void VDMS::CSVParserUtil::splitRowOnComma(const std::string &row, std::vector &rowvec) +{ + std::string::size_type start = 0; + while (start != std::string::npos) + { + std::string::size_type end = row.find(',', start); + if (end == std::string::npos) + { + if (start < row.size()) + { + rowvec.emplace_back(std::move(row.substr(start))); + } + break; + } + if (end > start) + { + rowvec.emplace_back(std::move(row.substr(start, end - start))); + } + start = end + 1; + } +} + +VDMS::CSVParserUtil::DATATYPE VDMS::CSVParserUtil ::isValidDataType(string data, int type) +{ + CSVParserUtil::DATATYPE actualtype = getDataType(data, ""); + return actualtype; + + // if(type==2 && (actualt=3 || acype=tualtype==4))//2 is for num + // return actualtype; + // else if(type==1 && (actualtype==1 || actualtype==2))//1 is for bool + // return actualtype; + // else + // return -1; +} + +bool VDMS::CSVParserUtil::isValidOpsType(string &type) +{ + if (type == "resize" || type == "threshold" || type == "flip" || type == "rotate" || type == "interval" || type == "crop") + return true; + return false; +} + +void VDMS::CSVParserUtil::parseBlobFile(const std::string &filename, std::string **descriptor_ptr) +{ + std::vector v; + + std::ifstream input(filename); + if (!input.is_open()) + { + // handle error if file cannot be opened + std::cerr << "Error: Could not open file " << filename << std::endl; + *descriptor_ptr = nullptr; + return; + } + + std::string str; + input >> str; + + std::stringstream ss(str); + while (ss.good()) + { + std::string substr; + getline(ss, substr, ';'); + v.push_back(std::stof(substr)); + } + + input.close(); + + // Convert vector to array of bytes + const size_t byteSize = v.size() * sizeof(float); + unsigned char *bytes = new unsigned char[byteSize]; + std::memcpy(bytes, v.data(), byteSize); + + // Copy bytes to dynamically allocated string + *descriptor_ptr = new std::string(reinterpret_cast(bytes), byteSize); + + // Clean up dynamically-allocated memory + delete[] bytes; +} + +void VDMS::CSVParserUtil::videoToString(const std::string &filename, std::string **video_data_ptr) +{ + // Open the video file in binary mode + std::ifstream file(filename, std::ios::binary); + + if (!file) + { + std::cerr << "Failed to open file: " << filename << std::endl; + *video_data_ptr = nullptr; + } + else + { + // Read the entire content of the file into a string + std::stringstream buffer; + buffer << file.rdbuf(); + std::string video_data = buffer.str(); + + *video_data_ptr = new std::string(video_data); + } + + // Close the file + file.close(); +} + +void VDMS::CSVParserUtil::read_blob_image(const std::string &filename, std::string **image_data_ptr) +{ + std::ifstream file(filename, std::ios::binary); + + if (file.is_open()) + { + + // Get the size of the file + file.seekg(0, std::ios::end); + size_t size = file.tellg(); + file.seekg(0, std::ios::beg); + + // Allocate a buffer to hold the file data + char *buffer = new char[size]; + + // Read the file data into the buffer + file.read(buffer, size); + + if (file.gcount() != size) + { + std::cerr << "Error: Failed to read entire file." << std::endl; + delete[] buffer; + *image_data_ptr = nullptr; + return; + } + + // Close the file + file.close(); + + // Allocate a new std::string to hold the image data + std::string *image_data = new std::string; + image_data->assign(buffer, size); + + // Free the buffer + delete[] buffer; + + // Assign the std::string pointer to the image_data_ptr + *image_data_ptr = image_data; + } + else + { + std::cerr << "Error: Failed to open file." << std::endl; + *image_data_ptr = nullptr; + } +} +VDMS::Response VDMS::CSVParserUtil::send_to_vdms(const Json::Value &query, + const std::vector blobs) +{ + Json::StyledWriter _fastwriter; + + return vdms_client->query(_fastwriter.write(query), blobs); +} diff --git a/client/cpp/CSVParserUtil.h b/client/cpp/CSVParserUtil.h new file mode 100644 index 00000000..ad30bd9d --- /dev/null +++ b/client/cpp/CSVParserUtil.h @@ -0,0 +1,102 @@ +#pragma once +#include +#include +#include +#include "rapidcsv.h" +#include +#include +#include +#include +#include +#include "VDMSClient.h" + +using namespace std; +using namespace std::chrono; + +namespace VDMS { +class CSVParserUtil { + + enum QueryType { EntityClass, + ConnectionClass, + ImagePath, + VideoPath, + DescriptorType, + DescriptorClass, + RectangleBound, + EntityUpdate, + ConnectionUpdate, + ImageUpdate, + RectangleUpdate + }; + enum class commandType { + AddEntity, + AddConnection, + AddImage, + AddVideo, + AddDescriptorSet, + AddDescriptor, + AddBoundingBox, + UpdateEntity, + UpdateConnection, + UpdateImage, + UpdateBoundingBox, + UNKNOWN + + }; + enum class DATATYPE{ + TRUE, + FALSE, + INTEGER, + FLOAT, + STRING, + DATE, + WRONG + + }; + std::map commands; + std::map command_list; + + + + public: + CSVParserUtil(); + CSVParserUtil(const std::string&, int port, const std::vector, int id ); + void initCommandsMap(); + int isBool( const string& data); + bool isFloat(const string &); + bool isInt(const string &); + VDMS::Response parse_row(std::vector& fields); + commandType get_query_type(const string &data); + CSVParserUtil::DATATYPE getDataType(const string& data,const string& colname); + void parseProperty(const string& columnNames,const string& row,const string& queryType, Json::Value &aquery); + void parseConstraints(const string &columnNames,const string& row, string& queryType, Json::Value &aquery); + void parseOperations(const string columnNames,string row,string queryType,Json::Value &aquery); + + // void parseCSVdata(int startrowcount,int endcount,rapidcsv::Document &doc, int &); + vector spiltrow(const string& row); + bool isValidOpsType(string& type); + void splitRowOnComma(const std::string& row, std::vector& rowvec); + + string function_accessing_columnNames(int i); + DATATYPE isValidDataType(string data,int type); + void read_blob_image(const std::string& filename, std::string** image_data_ptr); + void videoToString(const std::string& filename, std::string** video_data); + void parseBlobFile(const std::string& filename, std::string** descriptor_ptr); + + VDMS::Response send_to_vdms(const Json::Value &json_query, const std::vector blobs = {}); + + public: + std::string vdms_server; + int vdms_port; + std::vector _columnNames; + std::mutex querytype_mutex; + std::mutex aquery_mutex; + std::mutex cons_mutex; + std::mutex ops_mutex; + int id; + std::unique_ptr vdms_client; + + }; +}; + + diff --git a/client/cpp/ConnectionQueryParser.h b/client/cpp/ConnectionQueryParser.h new file mode 100644 index 00000000..afd8394d --- /dev/null +++ b/client/cpp/ConnectionQueryParser.h @@ -0,0 +1,115 @@ +#include "CSVParserUtil.h" +namespace VDMS { +class ConnectionQueryParser : public CSVParserUtil{ + public: + VDMS::Response ParseAddConnection(vector row, vector & cols); + // VDMS::Response ParseUpdateConnection(vector row, vector & cols); +}; +}; + +VDMS::Response VDMS::ConnectionQueryParser::ParseAddConnection(vector row, vector & columnNames){ + Json::Value aquery; + Json::Value allquery; + Json::Value find_query1, find_query2,find_query; + + if (row[0].empty()) { + std::cerr << "Error: Connection Class not provided\n"; + +} + +// Set command name and connection class +const string command_name = "AddConnection"; +const string connection_class = row[0]; +int ref1=1, ref2=3; +aquery["AddConnection"]["class"] = connection_class; + +// Parse class1 and class2 columns +for (int i = 1; i < columnNames.size(); i++) { + string column_name = columnNames[i]; + string column_value = row[i]; + string column_type = column_name.substr(0, 5); + string command="FindEntity"; + + + if (column_value.empty()) { + continue; + } + + + + if (column_name.find('@') != std::string::npos) { + std::size_t at_pos = column_name.find('@'); + + if (at_pos != std::string::npos) { + // Extract the name and id substrings. + std::string class1 = column_name.substr(0, at_pos); + std::string class_prop = column_name.substr(at_pos + 1); + + find_query["FindEntity"]["class"]=class1; + find_query["FindEntity"]["_ref"]=ref1++; + find_query["FindEntity"]["constraints"][class_prop][0] = "=="; + find_query["FindEntity"]["constraints"][class_prop][1]=column_value; + } + allquery.append(find_query); + + } + + + + // if (column_type == "cons_") { + // parseConstraints(column_name, column_value, command, find_query1); + // parseConstraints(column_name, column_value, command, find_query2); + // } + // if (column_type == "prop_") { + // parseProperty(column_name, column_value, command_name, aquery); + // } + find_query.clear(); + + +} + +// Set connection references +aquery["AddConnection"]["ref1"] = allquery[0]["FindEntity"]["_ref"]; +aquery["AddConnection"]["ref2"] = allquery[1]["FindEntity"]["_ref"]; + + + +allquery.append(aquery); + + +return send_to_vdms(allquery); +} + +// VDMS::Response VDMS::ConnectionQueryParser::ParseUpdateConnection(vector row, vector & columnNames){ +// Json::Value aquery; +// Json::Value aqueryf; +// Json::Value allquery; +// if(row[0]=="") +// throw "Connection Class not provided"; +// std::string command_name="UpdateConnection"; +// aquery["UpdateConnection"]["class"]=row[0]; +// aquery["UpdateConnection"]["_ref"]=96; +// aqueryf["FindConnection"]["class"]=row[0]; +// aqueryf["FindConnection"]["_ref"]=96; +// for(int j=1;j rowvec; +// splitRowOnComma(row[j],rowvec); +// for(int v=0;v row, vector& columnNames, int id); + +}; +}; + +VDMS::Response VDMS::DescriptorQueryParser::ParseAddDescriptor(vector row, vector& columnNames, int id){ + + if(row[0]==""){ + throw "Set not provided"; + } + Json::Value aquery; + Json::Value fullquery; + std::vector blobs; + std::string* descriptor; + std::string command_name="AddDescriptor"; + aquery["AddDescriptor"]["set"]=row[0]; + aquery["AddDescriptor"]["_ref"]=id+3; + for(int j=1;j row, vector& columnNames); + bool isValidMetric(string& metric); + bool isValidEngine(string& engine); +}; +}; +VDMS::Response VDMS::DescriptorSetQueryParser::ParseAddDescriptorSet(vector row, vector& columnNames){ + if(row[0]==""){ + throw "Descriptor Name not provided"; + } + Json::Value aquery; + Json::Value fullquery; + std::string command_name="AddDescriptorSet"; + aquery["AddDescriptorSet"]["name"]=row[0]; + + + for(int j=1;j + +namespace VDMS +{ + + class EntityQueryParser : public CSVParserUtil + { + public: + VDMS::Response ParseAddEntity(vector row, vector &cols); + // VDMS::Response ParseUpdateEntity(vector row, vector & cols); + }; +}; + +VDMS::Response VDMS::EntityQueryParser::ParseAddEntity(vector row, vector &cols) +{ + Json::Value aquery; + Json::Value fullquery; + + std::string command_name = "AddEntity"; + // std::cout << command_name << columnNames.size() < row, vector & cols){ +// Json:: Value aquery; +// Json::Value all_query; +// Json::Value find_query; +// std::string command_name="UpdateEntity"; +// if(row[0]==""){ +// throw "Entity Class not specified"; +// } +// aquery["UpdateEntity"]["class"]=row[0]; +// int ref=10; +// std::cout << _columnNames[0] < rowvec; +// splitRowOnComma(row[j],rowvec); +// for(int v=0;v row, vector columnNames); + // VDMS::Response ParseUpdateImage(vector row, vector columnNames); + bool ValidImageFormat(string data); + + +}; +}; + + + +VDMS::Response VDMS::ImageQueryParser::ParseAddImage(vector row, vector columnNames){ + Json::Value aquery; + Json::Value fullquery; + std::vector blobs; + // + if(row[0].empty()) + throw "Image path is not specified"; + if (columnNames.size() == 0) { + throw std::invalid_argument("Error: Column names vector is empty."); + } + + std::string command_name="AddImage"; + + aquery["AddImage"]["_ref"]=11; + + std::string name =row[0]; + + std::string* image_data_ptr = nullptr; + + read_blob_image(name, &image_data_ptr); + + // std::cout << *image_data_ptr << std::endl; + if(image_data_ptr!=nullptr){ + blobs.push_back(image_data_ptr); + // std::cout <<*blobs[0] < row, vector columnNames){ +// Json :: Value aquery; + +// Json::Value fullquery; + +// std::string command_name="UpdateIamge"; +// aquery["UpdateImage"]["_ref"]=12; +// for(int j=1;j rowvec; +// splitRowOnComma(row[j],rowvec); +// for(int v=0;v(end - start); +// cout << "duaration in ms is "< blobs) { diff --git a/client/cpp/VDMSClient.h b/client/cpp/VDMSClient.h index 866a9b8e..ced571d7 100644 --- a/client/cpp/VDMSClient.h +++ b/client/cpp/VDMSClient.h @@ -32,6 +32,7 @@ #include #include #include "comm/Connection.h" +// #include "CSVParser.h" namespace VDMS { @@ -39,6 +40,7 @@ namespace VDMS { std::string json; std::vector blobs; }; + class VDMSClient { static const int VDMS_PORT = 55555; @@ -49,13 +51,15 @@ namespace VDMS { // will leave the functioning like that. If the client has a need to // disconnect and connect specifically, then we can add explicit calls. comm::ConnClient _conn; + public: VDMSClient(std::string addr = "localhost", int port = VDMS_PORT); // Blocking call VDMS::Response query(const std::string &json_query, - const std::vector blobs = {}); - + const std::vector blobs = {}); + // void parse_csv_file(std::string filename, std::string , int); + }; }; diff --git a/client/cpp/VideoQueryParser.h b/client/cpp/VideoQueryParser.h new file mode 100644 index 00000000..3c89758b --- /dev/null +++ b/client/cpp/VideoQueryParser.h @@ -0,0 +1,84 @@ +#pragma once +#include "CSVParserUtil.h" +namespace VDMS { +class VideoQueryParser : public CSVParserUtil{ + public: + VDMS::Response ParseAddVideo(vector row, vector& columnNames); + bool isValidCodec(string& row); + bool isValidContainer(string& row); + +}; +} +VDMS::Response VDMS::VideoQueryParser::ParseAddVideo(vector row, vector& columnNames){ + Json::Value aquery; + Json::Value fullquery; + std::vector blobs; + if(row[0]=="") + throw "Video not provided"; + std::string command_name="AddVideo"; + + std::string video_name=row[0]; + try { + std::string* video_data_ptr; + CSVParserUtil::videoToString(video_name, &video_data_ptr); + + if(video_data_ptr!=nullptr){ + blobs.push_back(video_data_ptr); + // std::cout <<*blobs[0] < +#include +#include +#ifdef HAS_CODECVT +#include +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif + +namespace rapidcsv +{ +#if defined(_MSC_VER) + static const bool sPlatformHasCR = true; +#else + static const bool sPlatformHasCR = false; +#endif + + /** + * @brief Datastructure holding parameters controlling how invalid numbers (including + * empty strings) should be handled. + */ + struct ConverterParams + { + /** + * @brief Constructor + * @param pHasDefaultConverter specifies if conversion of non-numerical strings shall be + * converted to a default numerical value, instead of causing + * an exception to be thrown (default). + * @param pDefaultFloat floating-point default value to represent invalid numbers. + * @param pDefaultInteger integer default value to represent invalid numbers. + */ + explicit ConverterParams(const bool pHasDefaultConverter = false, + const long double pDefaultFloat = std::numeric_limits::signaling_NaN(), + const long long pDefaultInteger = 0) + : mHasDefaultConverter(pHasDefaultConverter) + , mDefaultFloat(pDefaultFloat) + , mDefaultInteger(pDefaultInteger) + { + } + + /** + * @brief specifies if conversion of non-numerical strings shall be converted to a default + * numerical value, instead of causing an exception to be thrown (default). + */ + bool mHasDefaultConverter; + + /** + * @brief floating-point default value to represent invalid numbers. + */ + long double mDefaultFloat; + + /** + * @brief integer default value to represent invalid numbers. + */ + long long mDefaultInteger; + }; + + /** + * @brief Exception thrown when attempting to access Document data in a datatype which + * is not supported by the Converter class. + */ + class no_converter : public std::exception + { + /** + * @brief Provides details about the exception + * @returns an explanatory string + */ + virtual const char* what() const throw() + { + return "unsupported conversion datatype"; + } + }; + + /** + * @brief Class providing conversion to/from numerical datatypes and strings. Only + * intended for rapidcsv internal usage, but exposed externally to allow + * specialization for custom datatype conversions. + */ + template + class Converter + { + public: + /** + * @brief Constructor + * @param pConverterParams specifies how conversion of non-numerical values to + * numerical datatype shall be handled. + */ + Converter(const ConverterParams& pConverterParams) + : mConverterParams(pConverterParams) + { + } + + /** + * @brief Converts numerical value to string representation. + * @param pVal numerical value + * @param pStr output string + */ + void ToStr(const T& pVal, std::string& pStr) const + { + if (typeid(T) == typeid(int) || + typeid(T) == typeid(long) || + typeid(T) == typeid(long long) || + typeid(T) == typeid(unsigned) || + typeid(T) == typeid(unsigned long) || + typeid(T) == typeid(unsigned long long) || + typeid(T) == typeid(float) || + typeid(T) == typeid(double) || + typeid(T) == typeid(long double) || + typeid(T) == typeid(char)) + { + std::ostringstream out; + out << pVal; + pStr = out.str(); + } + else + { + throw no_converter(); + } + } + + /** + * @brief Converts string holding a numerical value to numerical datatype representation. + * @param pVal numerical value + * @param pStr output string + */ + void ToVal(const std::string& pStr, T& pVal) const + { + try + { + if (typeid(T) == typeid(int)) + { + pVal = static_cast(std::stoi(pStr)); + return; + } + else if (typeid(T) == typeid(long)) + { + pVal = static_cast(std::stol(pStr)); + return; + } + else if (typeid(T) == typeid(long long)) + { + pVal = static_cast(std::stoll(pStr)); + return; + } + else if (typeid(T) == typeid(unsigned)) + { + pVal = static_cast(std::stoul(pStr)); + return; + } + else if (typeid(T) == typeid(unsigned long)) + { + pVal = static_cast(std::stoul(pStr)); + return; + } + else if (typeid(T) == typeid(unsigned long long)) + { + pVal = static_cast(std::stoull(pStr)); + return; + } + } + catch (...) + { + if (!mConverterParams.mHasDefaultConverter) + { + throw; + } + else + { + pVal = static_cast(mConverterParams.mDefaultInteger); + return; + } + } + + try + { + if (typeid(T) == typeid(float)) + { + pVal = static_cast(std::stof(pStr)); + return; + } + else if (typeid(T) == typeid(double)) + { + pVal = static_cast(std::stod(pStr)); + return; + } + else if (typeid(T) == typeid(long double)) + { + pVal = static_cast(std::stold(pStr)); + return; + } + } + catch (...) + { + if (!mConverterParams.mHasDefaultConverter) + { + throw; + } + else + { + pVal = static_cast(mConverterParams.mDefaultFloat); + return; + } + } + + if (typeid(T) == typeid(char)) + { + pVal = static_cast(pStr[0]); + return; + } + else + { + throw no_converter(); + } + } + + private: + const ConverterParams& mConverterParams; + }; + + /** + * @brief Specialized implementation handling string to string conversion. + * @param pVal string + * @param pStr string + */ + template<> + inline void Converter::ToStr(const std::string& pVal, std::string& pStr) const + { + pStr = pVal; + } + + /** + * @brief Specialized implementation handling string to string conversion. + * @param pVal string + * @param pStr string + */ + template<> + inline void Converter::ToVal(const std::string& pStr, std::string& pVal) const + { + pVal = pStr; + } + + template + using ConvFunc = std::function; + + /** + * @brief Datastructure holding parameters controlling which row and column should be + * treated as labels. + */ + struct LabelParams + { + /** + * @brief Constructor + * @param pColumnNameIdx specifies the zero-based row index of the column labels, setting + * it to -1 prevents column lookup by label name, and gives access + * to all rows as document data. Default: 0 + * @param pRowNameIdx specifies the zero-based column index of the row labels, setting + * it to -1 prevents row lookup by label name, and gives access + * to all columns as document data. Default: -1 + */ + explicit LabelParams(const int pColumnNameIdx = 0, const int pRowNameIdx = -1) + : mColumnNameIdx(pColumnNameIdx) + , mRowNameIdx(pRowNameIdx) + { + } + + /** + * @brief specifies the zero-based row index of the column labels. + */ + int mColumnNameIdx; + + /** + * @brief specifies the zero-based column index of the row labels. + */ + int mRowNameIdx; + }; + + /** + * @brief Datastructure holding parameters controlling how the CSV data fields are separated. + */ + struct SeparatorParams + { + /** + * @brief Constructor + * @param pSeparator specifies the column separator (default ','). + * @param pTrim specifies whether to trim leading and trailing spaces from + * cells read (default false). + * @param pHasCR specifies whether a new document (i.e. not an existing document read) + * should use CR/LF instead of only LF (default is to use standard + * behavior of underlying platforms - CR/LF for Win, and LF for others). + * @param pQuotedLinebreaks specifies whether to allow line breaks in quoted text (default false) + * @param pAutoQuote specifies whether to automatically dequote data during read, and add + * quotes during write (default true). + */ + explicit SeparatorParams(const char pSeparator = ',', const bool pTrim = false, + const bool pHasCR = sPlatformHasCR, const bool pQuotedLinebreaks = false, + const bool pAutoQuote = true) + : mSeparator(pSeparator) + , mTrim(pTrim) + , mHasCR(pHasCR) + , mQuotedLinebreaks(pQuotedLinebreaks) + , mAutoQuote(pAutoQuote) + { + } + + /** + * @brief specifies the column separator. + */ + char mSeparator; + + /** + * @brief specifies whether to trim leading and trailing spaces from cells read. + */ + bool mTrim; + + /** + * @brief specifies whether new documents should use CR/LF instead of LF. + */ + bool mHasCR; + + /** + * @brief specifies whether to allow line breaks in quoted text. + */ + bool mQuotedLinebreaks; + + /** + * @brief specifies whether to automatically dequote cell data. + */ + bool mAutoQuote; + }; + + /** + * @brief Datastructure holding parameters controlling how special line formats should be + * treated. + */ + struct LineReaderParams + { + /** + * @brief Constructor + * @param pSkipCommentLines specifies whether to skip lines prefixed with + * mCommentPrefix. Default: false + * @param pCommentPrefix specifies which prefix character to indicate a comment + * line. Default: # + * @param pSkipEmptyLines specifies whether to skip empty lines. Default: false + */ + explicit LineReaderParams(const bool pSkipCommentLines = false, + const char pCommentPrefix = '#', + const bool pSkipEmptyLines = false) + : mSkipCommentLines(pSkipCommentLines) + , mCommentPrefix(pCommentPrefix) + , mSkipEmptyLines(pSkipEmptyLines) + { + } + + /** + * @brief specifies whether to skip lines prefixed with mCommentPrefix. + */ + bool mSkipCommentLines; + + /** + * @brief specifies which prefix character to indicate a comment line. + */ + char mCommentPrefix; + + /** + * @brief specifies whether to skip empty lines. + */ + bool mSkipEmptyLines; + }; + + /** + * @brief Class representing a CSV document. + */ + class Document + { + public: + /** + * @brief Constructor + * @param pPath specifies the path of an existing CSV-file to populate the Document + * data with. + * @param pLabelParams specifies which row and column should be treated as labels. + * @param pSeparatorParams specifies which field and row separators should be used. + * @param pConverterParams specifies how invalid numbers (including empty strings) should be + * handled. + * @param pLineReaderParams specifies how special line formats should be treated. + */ + explicit Document(const std::string& pPath = std::string(), + const LabelParams& pLabelParams = LabelParams(), + const SeparatorParams& pSeparatorParams = SeparatorParams(), + const ConverterParams& pConverterParams = ConverterParams(), + const LineReaderParams& pLineReaderParams = LineReaderParams()) + : mPath(pPath) + , mLabelParams(pLabelParams) + , mSeparatorParams(pSeparatorParams) + , mConverterParams(pConverterParams) + , mLineReaderParams(pLineReaderParams) + { + if (!mPath.empty()) + { + ReadCsv(); + } + } + + /** + * @brief Constructor + * @param pStream specifies an input stream to read CSV data from. + * @param pLabelParams specifies which row and column should be treated as labels. + * @param pSeparatorParams specifies which field and row separators should be used. + * @param pConverterParams specifies how invalid numbers (including empty strings) should be + * handled. + * @param pLineReaderParams specifies how special line formats should be treated. + */ + explicit Document(std::istream& pStream, + const LabelParams& pLabelParams = LabelParams(), + const SeparatorParams& pSeparatorParams = SeparatorParams(), + const ConverterParams& pConverterParams = ConverterParams(), + const LineReaderParams& pLineReaderParams = LineReaderParams()) + : mPath() + , mLabelParams(pLabelParams) + , mSeparatorParams(pSeparatorParams) + , mConverterParams(pConverterParams) + , mLineReaderParams(pLineReaderParams) + { + ReadCsv(pStream); + } + + /** + * @brief Read Document data from file. + * @param pPath specifies the path of an existing CSV-file to populate the Document + * data with. + * @param pLabelParams specifies which row and column should be treated as labels. + * @param pSeparatorParams specifies which field and row separators should be used. + * @param pConverterParams specifies how invalid numbers (including empty strings) should be + * handled. + * @param pLineReaderParams specifies how special line formats should be treated. + */ + void Load(const std::string& pPath, + const LabelParams& pLabelParams = LabelParams(), + const SeparatorParams& pSeparatorParams = SeparatorParams(), + const ConverterParams& pConverterParams = ConverterParams(), + const LineReaderParams& pLineReaderParams = LineReaderParams()) + { + mPath = pPath; + mLabelParams = pLabelParams; + mSeparatorParams = pSeparatorParams; + mConverterParams = pConverterParams; + mLineReaderParams = pLineReaderParams; + ReadCsv(); + } + + /** + * @brief Read Document data from stream. + * @param pStream specifies an input stream to read CSV data from. + * @param pLabelParams specifies which row and column should be treated as labels. + * @param pSeparatorParams specifies which field and row separators should be used. + * @param pConverterParams specifies how invalid numbers (including empty strings) should be + * handled. + * @param pLineReaderParams specifies how special line formats should be treated. + */ + void Load(std::istream& pStream, + const LabelParams& pLabelParams = LabelParams(), + const SeparatorParams& pSeparatorParams = SeparatorParams(), + const ConverterParams& pConverterParams = ConverterParams(), + const LineReaderParams& pLineReaderParams = LineReaderParams()) + { + mPath = ""; + mLabelParams = pLabelParams; + mSeparatorParams = pSeparatorParams; + mConverterParams = pConverterParams; + mLineReaderParams = pLineReaderParams; + ReadCsv(pStream); + } + + /** + * @brief Write Document data to file. + * @param pPath optionally specifies the path where the CSV-file will be created + * (if not specified, the original path provided when creating or + * loading the Document data will be used). + */ + void Save(const std::string& pPath = std::string()) + { + if (!pPath.empty()) + { + mPath = pPath; + } + WriteCsv(); + } + + /** + * @brief Write Document data to stream. + * @param pStream specifies an output stream to write the data to. + */ + void Save(std::ostream& pStream) + { + WriteCsv(pStream); + } + + /** + * @brief Clears loaded Document data. + * + */ + void Clear() + { + mData.clear(); + mColumnNames.clear(); + mRowNames.clear(); +#ifdef HAS_CODECVT + mIsUtf16 = false; + mIsLE = false; +#endif + } + + /** + * @brief Get column index by name. + * @param pColumnName column label name. + * @returns zero-based column index. + */ + ssize_t GetColumnIdx(const std::string& pColumnName) const + { + if (mLabelParams.mColumnNameIdx >= 0) + { + if (mColumnNames.find(pColumnName) != mColumnNames.end()) + { + return mColumnNames.at(pColumnName) - (mLabelParams.mRowNameIdx + 1); + } + } + return -1; + } + + /** + * @brief Get column by index. + * @param pColumnIdx zero-based column index. + * @returns vector of column data. + */ + template + std::vector GetColumn(const size_t pColumnIdx) const + { + const ssize_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + std::vector column; + Converter converter(mConverterParams); + for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) + { + if (std::distance(mData.begin(), itRow) > mLabelParams.mColumnNameIdx) + { + if (columnIdx < static_cast(itRow->size())) + { + T val; + converter.ToVal(itRow->at(columnIdx), val); + column.push_back(val); + } + else + { + const std::string errStr = "requested column index " + + std::to_string(columnIdx - (mLabelParams.mRowNameIdx + 1)) + " >= " + + std::to_string(itRow->size() - (mLabelParams.mRowNameIdx + 1)) + + " (number of columns on row index " + + std::to_string(std::distance(mData.begin(), itRow) - + (mLabelParams.mColumnNameIdx + 1)) + ")"; + throw std::out_of_range(errStr); + } + } + } + return column; + } + + /** + * @brief Get column by index. + * @param pColumnIdx zero-based column index. + * @param pToVal conversion function. + * @returns vector of column data. + */ + template + std::vector GetColumn(const size_t pColumnIdx, ConvFunc pToVal) const + { + const ssize_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + std::vector column; + for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) + { + if (std::distance(mData.begin(), itRow) > mLabelParams.mColumnNameIdx) + { + T val; + pToVal(itRow->at(columnIdx), val); + column.push_back(val); + } + } + return column; + } + + /** + * @brief Get column by name. + * @param pColumnName column label name. + * @returns vector of column data. + */ + template + std::vector GetColumn(const std::string& pColumnName) const + { + const ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + return GetColumn(columnIdx); + } + + /** + * @brief Get column by name. + * @param pColumnName column label name. + * @param pToVal conversion function. + * @returns vector of column data. + */ + template + std::vector GetColumn(const std::string& pColumnName, ConvFunc pToVal) const + { + const ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + return GetColumn(columnIdx, pToVal); + } + + /** + * @brief Set column by index. + * @param pColumnIdx zero-based column index. + * @param pColumn vector of column data. + */ + template + void SetColumn(const size_t pColumnIdx, const std::vector& pColumn) + { + const size_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + + while (pColumn.size() + (mLabelParams.mColumnNameIdx + 1) > GetDataRowCount()) + { + std::vector row; + row.resize(GetDataColumnCount()); + mData.push_back(row); + } + + if ((columnIdx + 1) > GetDataColumnCount()) + { + for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) + { + itRow->resize(columnIdx + 1 + (mLabelParams.mRowNameIdx + 1)); + } + } + + Converter converter(mConverterParams); + for (auto itRow = pColumn.begin(); itRow != pColumn.end(); ++itRow) + { + std::string str; + converter.ToStr(*itRow, str); + mData.at(std::distance(pColumn.begin(), itRow) + (mLabelParams.mColumnNameIdx + 1)).at(columnIdx) = str; + } + } + + /** + * @brief Set column by name. + * @param pColumnName column label name. + * @param pColumn vector of column data. + */ + template + void SetColumn(const std::string& pColumnName, const std::vector& pColumn) + { + const ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + SetColumn(columnIdx, pColumn); + } + + /** + * @brief Remove column by index. + * @param pColumnIdx zero-based column index. + */ + void RemoveColumn(const size_t pColumnIdx) + { + const ssize_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) + { + itRow->erase(itRow->begin() + columnIdx); + } + } + + /** + * @brief Remove column by name. + * @param pColumnName column label name. + */ + void RemoveColumn(const std::string& pColumnName) + { + ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + + RemoveColumn(columnIdx); + } + + /** + * @brief Insert column at specified index. + * @param pColumnIdx zero-based column index. + * @param pColumn vector of column data (optional argument). + * @param pColumnName column label name (optional argument). + */ + template + void InsertColumn(const size_t pColumnIdx, const std::vector& pColumn = std::vector(), + const std::string& pColumnName = std::string()) + { + const size_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + + std::vector column; + if (pColumn.empty()) + { + column.resize(GetDataRowCount()); + } + else + { + column.resize(pColumn.size() + (mLabelParams.mColumnNameIdx + 1)); + Converter converter(mConverterParams); + for (auto itRow = pColumn.begin(); itRow != pColumn.end(); ++itRow) + { + std::string str; + converter.ToStr(*itRow, str); + const size_t rowIdx = std::distance(pColumn.begin(), itRow) + (mLabelParams.mColumnNameIdx + 1); + column.at(rowIdx) = str; + } + } + + while (column.size() > GetDataRowCount()) + { + std::vector row; + const size_t columnCount = std::max(static_cast(mLabelParams.mColumnNameIdx + 1), + GetDataColumnCount()); + row.resize(columnCount); + mData.push_back(row); + } + + for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) + { + const size_t rowIdx = std::distance(mData.begin(), itRow); + itRow->insert(itRow->begin() + columnIdx, column.at(rowIdx)); + } + + if (!pColumnName.empty()) + { + SetColumnName(pColumnIdx, pColumnName); + } + } + + /** + * @brief Get number of data columns (excluding label columns). + * @returns column count. + */ + size_t GetColumnCount() const + { + const ssize_t count = static_cast((mData.size() > 0) ? mData.at(0).size() : 0) - + (mLabelParams.mRowNameIdx + 1); + return (count >= 0) ? count : 0; + } + + /** + * @brief Get row index by name. + * @param pRowName row label name. + * @returns zero-based row index. + */ + ssize_t GetRowIdx(const std::string& pRowName) const + { + if (mLabelParams.mRowNameIdx >= 0) + { + if (mRowNames.find(pRowName) != mRowNames.end()) + { + return mRowNames.at(pRowName) - (mLabelParams.mColumnNameIdx + 1); + } + } + return -1; + } + + /** + * @brief Get row by index. + * @param pRowIdx zero-based row index. + * @returns vector of row data. + */ + template + std::vector GetRow(const size_t pRowIdx) const + { + const ssize_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + std::vector row; + Converter converter(mConverterParams); + for (auto itCol = mData.at(rowIdx).begin(); itCol != mData.at(rowIdx).end(); ++itCol) + { + if (std::distance(mData.at(rowIdx).begin(), itCol) > mLabelParams.mRowNameIdx) + { + T val; + converter.ToVal(*itCol, val); + row.push_back(val); + } + } + return row; + } + + /** + * @brief Get row by index. + * @param pRowIdx zero-based row index. + * @param pToVal conversion function. + * @returns vector of row data. + */ + template + std::vector GetRow(const size_t pRowIdx, ConvFunc pToVal) const + { + const ssize_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + std::vector row; + Converter converter(mConverterParams); + for (auto itCol = mData.at(rowIdx).begin(); itCol != mData.at(rowIdx).end(); ++itCol) + { + if (std::distance(mData.at(rowIdx).begin(), itCol) > mLabelParams.mRowNameIdx) + { + T val; + pToVal(*itCol, val); + row.push_back(val); + } + } + return row; + } + + /** + * @brief Get row by name. + * @param pRowName row label name. + * @returns vector of row data. + */ + template + std::vector GetRow(const std::string& pRowName) const + { + ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + return GetRow(rowIdx); + } + + /** + * @brief Get row by name. + * @param pRowName row label name. + * @param pToVal conversion function. + * @returns vector of row data. + */ + template + std::vector GetRow(const std::string& pRowName, ConvFunc pToVal) const + { + ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + return GetRow(rowIdx, pToVal); + } + + /** + * @brief Set row by index. + * @param pRowIdx zero-based row index. + * @param pRow vector of row data. + */ + template + void SetRow(const size_t pRowIdx, const std::vector& pRow) + { + const size_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + + while ((rowIdx + 1) > GetDataRowCount()) + { + std::vector row; + row.resize(GetDataColumnCount()); + mData.push_back(row); + } + + if (pRow.size() > GetDataColumnCount()) + { + for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) + { + itRow->resize(pRow.size() + (mLabelParams.mRowNameIdx + 1)); + } + } + + Converter converter(mConverterParams); + for (auto itCol = pRow.begin(); itCol != pRow.end(); ++itCol) + { + std::string str; + converter.ToStr(*itCol, str); + mData.at(rowIdx).at(std::distance(pRow.begin(), itCol) + (mLabelParams.mRowNameIdx + 1)) = str; + } + } + + /** + * @brief Set row by name. + * @param pRowName row label name. + * @param pRow vector of row data. + */ + template + void SetRow(const std::string& pRowName, const std::vector& pRow) + { + ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + return SetRow(rowIdx, pRow); + } + + /** + * @brief Remove row by index. + * @param pRowIdx zero-based row index. + */ + void RemoveRow(const size_t pRowIdx) + { + const ssize_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + mData.erase(mData.begin() + rowIdx); + } + + /** + * @brief Remove row by name. + * @param pRowName row label name. + */ + void RemoveRow(const std::string& pRowName) + { + ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + + RemoveRow(rowIdx); + } + + /** + * @brief Insert row at specified index. + * @param pRowIdx zero-based row index. + * @param pRow vector of row data (optional argument). + * @param pRowName row label name (optional argument). + */ + template + void InsertRow(const size_t pRowIdx, const std::vector& pRow = std::vector(), + const std::string& pRowName = std::string()) + { + const size_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + + std::vector row; + if (pRow.empty()) + { + row.resize(GetDataColumnCount()); + } + else + { + row.resize(pRow.size() + (mLabelParams.mRowNameIdx + 1)); + Converter converter(mConverterParams); + for (auto itCol = pRow.begin(); itCol != pRow.end(); ++itCol) + { + std::string str; + converter.ToStr(*itCol, str); + row.at(std::distance(pRow.begin(), itCol) + (mLabelParams.mRowNameIdx + 1)) = str; + } + } + + while (rowIdx > GetDataRowCount()) + { + std::vector tempRow; + tempRow.resize(GetDataColumnCount()); + mData.push_back(tempRow); + } + + mData.insert(mData.begin() + rowIdx, row); + + if (!pRowName.empty()) + { + SetRowName(pRowIdx, pRowName); + } + } + + /** + * @brief Get number of data rows (excluding label rows). + * @returns row count. + */ + size_t GetRowCount() const + { + const ssize_t count = static_cast(mData.size()) - (mLabelParams.mColumnNameIdx + 1); + return (count >= 0) ? count : 0; + } + + /** + * @brief Get cell by index. + * @param pColumnIdx zero-based column index. + * @param pRowIdx zero-based row index. + * @returns cell data. + */ + template + T GetCell(const size_t pColumnIdx, const size_t pRowIdx) const + { + const ssize_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + const ssize_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + + T val; + Converter converter(mConverterParams); + converter.ToVal(mData.at(rowIdx).at(columnIdx), val); + return val; + } + + /** + * @brief Get cell by index. + * @param pColumnIdx zero-based column index. + * @param pRowIdx zero-based row index. + * @param pToVal conversion function. + * @returns cell data. + */ + template + T GetCell(const size_t pColumnIdx, const size_t pRowIdx, ConvFunc pToVal) const + { + const ssize_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + const ssize_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + + T val; + pToVal(mData.at(rowIdx).at(columnIdx), val); + return val; + } + + /** + * @brief Get cell by name. + * @param pColumnName column label name. + * @param pRowName row label name. + * @returns cell data. + */ + template + T GetCell(const std::string& pColumnName, const std::string& pRowName) const + { + const ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + + const ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + + return GetCell(columnIdx, rowIdx); + } + + /** + * @brief Get cell by name. + * @param pColumnName column label name. + * @param pRowName row label name. + * @param pToVal conversion function. + * @returns cell data. + */ + template + T GetCell(const std::string& pColumnName, const std::string& pRowName, ConvFunc pToVal) const + { + const ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + + const ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + + return GetCell(columnIdx, rowIdx, pToVal); + } + + /** + * @brief Get cell by column name and row index. + * @param pColumnName column label name. + * @param pRowIdx zero-based row index. + * @returns cell data. + */ + template + T GetCell(const std::string& pColumnName, const size_t pRowIdx) const + { + const ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + + return GetCell(columnIdx, pRowIdx); + } + + /** + * @brief Get cell by column name and row index. + * @param pColumnName column label name. + * @param pRowIdx zero-based row index. + * @param pToVal conversion function. + * @returns cell data. + */ + template + T GetCell(const std::string& pColumnName, const size_t pRowIdx, ConvFunc pToVal) const + { + const ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + + return GetCell(columnIdx, pRowIdx, pToVal); + } + + /** + * @brief Get cell by column index and row name. + * @param pColumnIdx zero-based column index. + * @param pRowName row label name. + * @returns cell data. + */ + template + T GetCell(const size_t pColumnIdx, const std::string& pRowName) const + { + const ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + + return GetCell(pColumnIdx, rowIdx); + } + + /** + * @brief Get cell by column index and row name. + * @param pColumnIdx zero-based column index. + * @param pRowName row label name. + * @param pToVal conversion function. + * @returns cell data. + */ + template + T GetCell(const size_t pColumnIdx, const std::string& pRowName, ConvFunc pToVal) const + { + const ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + + return GetCell(pColumnIdx, rowIdx, pToVal); + } + + /** + * @brief Set cell by index. + * @param pRowIdx zero-based row index. + * @param pColumnIdx zero-based column index. + * @param pCell cell data. + */ + template + void SetCell(const size_t pColumnIdx, const size_t pRowIdx, const T& pCell) + { + const size_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + const size_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + + while ((rowIdx + 1) > GetDataRowCount()) + { + std::vector row; + row.resize(GetDataColumnCount()); + mData.push_back(row); + } + + if ((columnIdx + 1) > GetDataColumnCount()) + { + for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) + { + itRow->resize(columnIdx + 1); + } + } + + std::string str; + Converter converter(mConverterParams); + converter.ToStr(pCell, str); + mData.at(rowIdx).at(columnIdx) = str; + } + + /** + * @brief Set cell by name. + * @param pColumnName column label name. + * @param pRowName row label name. + * @param pCell cell data. + */ + template + void SetCell(const std::string& pColumnName, const std::string& pRowName, const T& pCell) + { + const ssize_t columnIdx = GetColumnIdx(pColumnName); + if (columnIdx < 0) + { + throw std::out_of_range("column not found: " + pColumnName); + } + + const ssize_t rowIdx = GetRowIdx(pRowName); + if (rowIdx < 0) + { + throw std::out_of_range("row not found: " + pRowName); + } + + SetCell(columnIdx, rowIdx, pCell); + } + + /** + * @brief Get column name + * @param pColumnIdx zero-based column index. + * @returns column name. + */ + std::string GetColumnName(const ssize_t pColumnIdx) + { + const ssize_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + if (mLabelParams.mColumnNameIdx < 0) + { + throw std::out_of_range("column name row index < 0: " + std::to_string(mLabelParams.mColumnNameIdx)); + } + + return mData.at(mLabelParams.mColumnNameIdx).at(columnIdx); + } + + /** + * @brief Set column name + * @param pColumnIdx zero-based column index. + * @param pColumnName column name. + */ + void SetColumnName(size_t pColumnIdx, const std::string& pColumnName) + { + const ssize_t columnIdx = pColumnIdx + (mLabelParams.mRowNameIdx + 1); + mColumnNames[pColumnName] = columnIdx; + if (mLabelParams.mColumnNameIdx < 0) + { + throw std::out_of_range("column name row index < 0: " + std::to_string(mLabelParams.mColumnNameIdx)); + } + + // increase table size if necessary: + const int rowIdx = mLabelParams.mColumnNameIdx; + if (rowIdx >= static_cast(mData.size())) + { + mData.resize(rowIdx + 1); + } + auto& row = mData[rowIdx]; + if (columnIdx >= static_cast(row.size())) + { + row.resize(columnIdx + 1); + } + + mData.at(mLabelParams.mColumnNameIdx).at(columnIdx) = pColumnName; + } + + /** + * @brief Get column names + * @returns vector of column names. + */ + std::vector GetColumnNames() + { + if (mLabelParams.mColumnNameIdx >= 0) + { + return std::vector(mData.at(mLabelParams.mColumnNameIdx).begin() + + (mLabelParams.mRowNameIdx + 1), + mData.at(mLabelParams.mColumnNameIdx).end()); + } + + return std::vector(); + } + + /** + * @brief Get row name + * @param pRowIdx zero-based column index. + * @returns row name. + */ + std::string GetRowName(const ssize_t pRowIdx) + { + const ssize_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + if (mLabelParams.mRowNameIdx < 0) + { + throw std::out_of_range("row name column index < 0: " + std::to_string(mLabelParams.mRowNameIdx)); + } + + return mData.at(rowIdx).at(mLabelParams.mRowNameIdx); + } + + /** + * @brief Set row name + * @param pRowIdx zero-based row index. + * @param pRowName row name. + */ + void SetRowName(size_t pRowIdx, const std::string& pRowName) + { + const ssize_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); + mRowNames[pRowName] = rowIdx; + if (mLabelParams.mRowNameIdx < 0) + { + throw std::out_of_range("row name column index < 0: " + std::to_string(mLabelParams.mRowNameIdx)); + } + + // increase table size if necessary: + if (rowIdx >= static_cast(mData.size())) + { + mData.resize(rowIdx + 1); + } + auto& row = mData[rowIdx]; + if (mLabelParams.mRowNameIdx >= static_cast(row.size())) + { + row.resize(mLabelParams.mRowNameIdx + 1); + } + + mData.at(rowIdx).at(mLabelParams.mRowNameIdx) = pRowName; + } + + /** + * @brief Get row names + * @returns vector of row names. + */ + std::vector GetRowNames() + { + std::vector rownames; + if (mLabelParams.mRowNameIdx >= 0) + { + for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) + { + if (std::distance(mData.begin(), itRow) > mLabelParams.mColumnNameIdx) + { + rownames.push_back(itRow->at(mLabelParams.mRowNameIdx)); + } + } + } + return rownames; + } + + private: + void ReadCsv() + { + std::ifstream stream; + stream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + stream.open(mPath, std::ios::binary); + ReadCsv(stream); + } + + void ReadCsv(std::istream& pStream) + { + Clear(); + pStream.seekg(0, std::ios::end); + std::streamsize length = pStream.tellg(); + pStream.seekg(0, std::ios::beg); + +#ifdef HAS_CODECVT + std::vector bom2b(2, '\0'); + if (length >= 2) + { + pStream.read(bom2b.data(), 2); + pStream.seekg(0, std::ios::beg); + } + + static const std::vector bomU16le = { '\xff', '\xfe' }; + static const std::vector bomU16be = { '\xfe', '\xff' }; + if ((bom2b == bomU16le) || (bom2b == bomU16be)) + { + mIsUtf16 = true; + mIsLE = (bom2b == bomU16le); + + std::wifstream wstream; + wstream.exceptions(std::wifstream::failbit | std::wifstream::badbit); + wstream.open(mPath, std::ios::binary); + if (mIsLE) + { + wstream.imbue(std::locale(wstream.getloc(), + new std::codecvt_utf16(std::consume_header | + std::little_endian)>)); + } + else + { + wstream.imbue(std::locale(wstream.getloc(), + new std::codecvt_utf16)); + } + std::wstringstream wss; + wss << wstream.rdbuf(); + std::string utf8 = ToString(wss.str()); + std::stringstream ss(utf8); + ParseCsv(ss, utf8.size()); + } + else +#endif + { + // check for UTF-8 Byte order mark and skip it when found + if (length >= 3) + { + std::vector bom3b(3, '\0'); + pStream.read(bom3b.data(), 3); + static const std::vector bomU8 = { '\xef', '\xbb', '\xbf' }; + if (bom3b != bomU8) + { + // file does not start with a UTF-8 Byte order mark + pStream.seekg(0, std::ios::beg); + } + else + { + // file did start with a UTF-8 Byte order mark, simply skip it + length -= 3; + } + } + + ParseCsv(pStream, length); + } + } + + void ParseCsv(std::istream& pStream, std::streamsize p_FileLength) + { + const std::streamsize bufLength = 64 * 1024; + std::vector buffer(bufLength); + std::vector row; + std::string cell; + bool quoted = false; + int cr = 0; + int lf = 0; + + while (p_FileLength > 0) + { + std::streamsize readLength = std::min(p_FileLength, bufLength); + pStream.read(buffer.data(), readLength); + for (int i = 0; i < readLength; ++i) + { + if (buffer[i] == '"') + { + if (cell.empty() || cell[0] == '"') + { + quoted = !quoted; + } + cell += buffer[i]; + } + else if (buffer[i] == mSeparatorParams.mSeparator) + { + if (!quoted) + { + row.push_back(Unquote(Trim(cell))); + cell.clear(); + } + else + { + cell += buffer[i]; + } + } + else if (buffer[i] == '\r') + { + if (mSeparatorParams.mQuotedLinebreaks && quoted) + { + cell += buffer[i]; + } + else + { + ++cr; + } + } + else if (buffer[i] == '\n') + { + if (mSeparatorParams.mQuotedLinebreaks && quoted) + { + cell += buffer[i]; + } + else + { + ++lf; + if (mLineReaderParams.mSkipEmptyLines && row.empty() && cell.empty()) + { + // skip empty line + } + else + { + row.push_back(Unquote(Trim(cell))); + + if (mLineReaderParams.mSkipCommentLines && !row.at(0).empty() && + (row.at(0)[0] == mLineReaderParams.mCommentPrefix)) + { + // skip comment line + } + else + { + mData.push_back(row); + } + + cell.clear(); + row.clear(); + quoted = false; + } + } + } + else + { + cell += buffer[i]; + } + } + p_FileLength -= readLength; + } + + // Handle last line without linebreak + if (!cell.empty() || !row.empty()) + { + row.push_back(Unquote(Trim(cell))); + cell.clear(); + mData.push_back(row); + row.clear(); + } + + // Assume CR/LF if at least half the linebreaks have CR + mSeparatorParams.mHasCR = (cr > (lf / 2)); + + // Set up column labels + if ((mLabelParams.mColumnNameIdx >= 0) && + (static_cast(mData.size()) > mLabelParams.mColumnNameIdx)) + { + int i = 0; + for (auto& columnName : mData[mLabelParams.mColumnNameIdx]) + { + mColumnNames[columnName] = i++; + } + } + + // Set up row labels + if ((mLabelParams.mRowNameIdx >= 0) && + (static_cast(mData.size()) > + (mLabelParams.mColumnNameIdx + 1))) + { + int i = 0; + for (auto& dataRow : mData) + { + if (static_cast(dataRow.size()) > mLabelParams.mRowNameIdx) + { + mRowNames[dataRow[mLabelParams.mRowNameIdx]] = i++; + } + } + } + } + + void WriteCsv() const + { +#ifdef HAS_CODECVT + if (mIsUtf16) + { + std::stringstream ss; + WriteCsv(ss); + std::string utf8 = ss.str(); + std::wstring wstr = ToWString(utf8); + + std::wofstream wstream; + wstream.exceptions(std::wofstream::failbit | std::wofstream::badbit); + wstream.open(mPath, std::ios::binary | std::ios::trunc); + + if (mIsLE) + { + wstream.imbue(std::locale(wstream.getloc(), + new std::codecvt_utf16(std::little_endian)>)); + } + else + { + wstream.imbue(std::locale(wstream.getloc(), + new std::codecvt_utf16)); + } + + wstream << static_cast(0xfeff); + wstream << wstr; + } + else +#endif + { + std::ofstream stream; + stream.exceptions(std::ofstream::failbit | std::ofstream::badbit); + stream.open(mPath, std::ios::binary | std::ios::trunc); + WriteCsv(stream); + } + } + + void WriteCsv(std::ostream& pStream) const + { + for (auto itr = mData.begin(); itr != mData.end(); ++itr) + { + for (auto itc = itr->begin(); itc != itr->end(); ++itc) + { + if (mSeparatorParams.mAutoQuote && + ((itc->find(mSeparatorParams.mSeparator) != std::string::npos) || + (itc->find(' ') != std::string::npos))) + { + // escape quotes in string + std::string str = *itc; + ReplaceString(str, "\"", "\"\""); + + pStream << "\"" << str << "\""; + } + else + { + pStream << *itc; + } + + if (std::distance(itc, itr->end()) > 1) + { + pStream << mSeparatorParams.mSeparator; + } + } + pStream << (mSeparatorParams.mHasCR ? "\r\n" : "\n"); + } + } + + size_t GetDataRowCount() const + { + return mData.size(); + } + + size_t GetDataColumnCount() const + { + return (mData.size() > 0) ? mData.at(0).size() : 0; + } + + std::string Trim(const std::string& pStr) + { + if (mSeparatorParams.mTrim) + { + std::string str = pStr; + + // ltrim + str.erase(str.begin(), std::find_if(str.begin(), str.end(), [](int ch) { return !isspace(ch); })); + + // rtrim + str.erase(std::find_if(str.rbegin(), str.rend(), [](int ch) { return !isspace(ch); }).base(), str.end()); + + return str; + } + else + { + return pStr; + } + } + + std::string Unquote(const std::string& pStr) + { + if (mSeparatorParams.mAutoQuote && (pStr.size() >= 2) && (pStr.front() == '"') && (pStr.back() == '"')) + { + // remove start/end quotes + std::string str = pStr.substr(1, pStr.size() - 2); + + // unescape quotes in string + ReplaceString(str, "\"\"", "\""); + + return str; + } + else + { + return pStr; + } + } + +#ifdef HAS_CODECVT +#if defined(_MSC_VER) +#pragma warning (disable: 4996) +#endif + static std::string ToString(const std::wstring& pWStr) + { + return std::wstring_convert, wchar_t>{ }.to_bytes(pWStr); + } + + static std::wstring ToWString(const std::string& pStr) + { + return std::wstring_convert, wchar_t>{ }.from_bytes(pStr); + } +#if defined(_MSC_VER) +#pragma warning (default: 4996) +#endif +#endif + + static void ReplaceString(std::string& pStr, const std::string& pSearch, const std::string& pReplace) + { + size_t pos = 0; + + while ((pos = pStr.find(pSearch, pos)) != std::string::npos) + { + pStr.replace(pos, pSearch.size(), pReplace); + pos += pReplace.size(); + } + } + + private: + std::string mPath; + LabelParams mLabelParams; + SeparatorParams mSeparatorParams; + ConverterParams mConverterParams; + LineReaderParams mLineReaderParams; + std::vector> mData; + std::map mColumnNames; + std::map mRowNames; +#ifdef HAS_CODECVT + bool mIsUtf16 = false; + bool mIsLE = false; +#endif + }; +} \ No newline at end of file diff --git a/docker/check-in/run_coverage_cpp.sh b/docker/check-in/run_coverage_cpp.sh index 029d2c01..90f67d66 100644 --- a/docker/check-in/run_coverage_cpp.sh +++ b/docker/check-in/run_coverage_cpp.sh @@ -4,8 +4,10 @@ chmod +x run_tests.sh ./run_tests.sh gcovr --root /vdms \ - -e /vdms/src/pmgd -e /vdms/build -e /vdms/tests --gcov-ignore-errors=no_working_dir_found \ - -f "/vdms/.*/.*\.cc" \ + -e /vdms/src/pmgd -e /vdms/build -e /vdms/distributed -e /vdms/tests \ + --gcov-ignore-parse-errors=negative_hits.warn_once_per_file \ + --gcov-ignore-errors=no_working_dir_found \ + -f "/vdms/.*/.*\.cc" -f "/vdms/.*/.*\.cpp" \ --exclude-unreachable-branches \ --txt=/vdms/tests/coverage_report/c_coverage_report.txt \ --xml-pretty --xml=/vdms/tests/coverage_report/c_coverage_report.xml diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9fd1754a..06c72f71 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -40,6 +40,7 @@ add_executable(unit_tests unit_tests/DescriptorSetReadFS_test.cc unit_tests/DescriptorSetStore_test.cc unit_tests/client_add_entity.cc + unit_tests/client_csv.cc unit_tests/meta_data.cc unit_tests/client_find_entities.cc unit_tests/client_image.cc diff --git a/tests/csv_samples/CSVformat100.csv b/tests/csv_samples/CSVformat100.csv new file mode 100644 index 00000000..3df655d9 --- /dev/null +++ b/tests/csv_samples/CSVformat100.csv @@ -0,0 +1,96 @@ +EntityClass,prop_name,prop_middlename,prop_lastname,prop_id,prop_date:dob,prop_height,prop_weight,prop_age,prop_has_dog,prop_Gender,prop_email,prop_Address,prop_City,cons_1 +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,False,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,date:dob==1968-07-22T12:45:12-08:00 +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,id==1234 +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,weight==70.5 +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur +Person,Rajendra,kumar,Maheshwari,1234,1968-07-22T12:45:12-08:00,163,70.5,55,TRUE,M,mymail.2@gmail.com,90 Mohini Bhawan,Jodhpur,City==Jodhpur diff --git a/tests/csv_samples/Descriptor.csv b/tests/csv_samples/Descriptor.csv new file mode 100644 index 00000000..2ef43646 --- /dev/null +++ b/tests/csv_samples/Descriptor.csv @@ -0,0 +1,6 @@ +DescriptorClass,label,prop_age,prop_gender,inputdata +Test_14096,Rocky,34,M,../tests/csv_samples/blob_1.txt +Test_14096,Rocky,34,M,../tests/csv_samples/blob_1.txt +Test_14096,Rocky,34,M,../tests/csv_samples/blob_1.txt +Test_14096,Rocky,34,M,../tests/csv_samples/blob_1.txt +Test_14096,Rocky,34,M,../tests/csv_samples/blob_1.txt diff --git a/tests/csv_samples/DescriptorSet.csv b/tests/csv_samples/DescriptorSet.csv new file mode 100644 index 00000000..cf9a3f5b --- /dev/null +++ b/tests/csv_samples/DescriptorSet.csv @@ -0,0 +1,7 @@ +DescriptorType,dimensions,distancemetric,searchengine +Test1024,1024,L2,FaissFlat +Test_14096,1024,L2,FaissFlat +Test1000,1000,L2,FaissFlat +Test100,100,L2,FaissFlat +Test128,128,IP,FaissIVFFlat +Test512,512,L2,TileDBDense diff --git a/tests/csv_samples/Image.csv b/tests/csv_samples/Image.csv new file mode 100644 index 00000000..37723900 --- /dev/null +++ b/tests/csv_samples/Image.csv @@ -0,0 +1,11 @@ +ImagePath,ops_threshold,ops_crop,ops_resize,ops_flip,ops_rotate,prop_type,prop_part,format,cons_1 +../tests/test_images/large1.jpg,350,,,,,,image1,jpg,part==image1 +../tests/test_images/large1.jpg,350,"0,0,224,224",,,,,image2,jpg, +../tests/test_images/large1.jpg,350,,"224,224",,,,image3,jpg, +../tests/test_images/large1.jpg,350,,,1,,,image4,jpg, +../tests/test_images/large1.jpg,350,,,,"45,false",,image5,jpg, +../tests/test_images/large1.jpg,350,,,,"75.5,false",,image6,jpg, +../tests/test_images/large1.jpg,350,,,,,,image7,png, +../tests/test_images/large1.jpg,,,,,,,image8,bin, +../tests/test_images/large1.jpg,350,,,,,,image9,png, +../tests/test_images/large1.jpg,,,,,,,image10,bin, diff --git a/tests/csv_samples/Rectangle.csv b/tests/csv_samples/Rectangle.csv new file mode 100644 index 00000000..cc0fec24 --- /dev/null +++ b/tests/csv_samples/Rectangle.csv @@ -0,0 +1,13 @@ +RectangleBound,prop_name +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 +"1,2,3,4",2 \ No newline at end of file diff --git a/tests/csv_samples/Video.csv b/tests/csv_samples/Video.csv new file mode 100644 index 00000000..a9a8f4f4 --- /dev/null +++ b/tests/csv_samples/Video.csv @@ -0,0 +1,6 @@ +VideoPath,format,compressto,prop_name,ops_resize,ops_interval +../tests/test_videos/Megamind.avi,avi,h264,Good,"200,175", +../tests/test_videos/Megamind.avi,avi,h264,Good,,"10,50,2" +../tests/test_videos/Megamind.avi,avi,h264,Good,, +../tests/test_videos/Megamind.avi,avi,h264,Good,, +../tests/test_videos/Megamind.avi,avi,h264,Good,, diff --git a/tests/csv_samples/blob_1.txt b/tests/csv_samples/blob_1.txt new file mode 100644 index 00000000..5bde9ca8 --- /dev/null +++ b/tests/csv_samples/blob_1.txt @@ -0,0 +1 @@ +0.840188;0.394383;0.783099;0.79844;0.911647;0.197551;0.335223;0.76823;0.277775;0.55397;0.477397;0.628871;0.364784;0.513401;0.95223;0.916195;0.635712;0.717297;0.141603;0.606969;0.0163006;0.242887;0.137232;0.804177;0.156679;0.400944;0.12979;0.108809;0.998924;0.218257;0.512932;0.839112;0.61264;0.296032;0.637552;0.524287;0.493583;0.972775;0.292517;0.771358;0.526745;0.769914;0.400229;0.891529;0.283315;0.352458;0.807725;0.919026;0.0697553;0.949327;0.525995;0.0860558;0.192214;0.663227;0.890233;0.348893;0.0641713;0.020023;0.457702;0.0630958;0.23828;0.970634;0.902208;0.85092;0.266666;0.53976;0.375207;0.760249;0.512535;0.667724;0.531606;0.0392803;0.437638;0.931835;0.93081;0.720952;0.284293;0.738534;0.639979;0.354049;0.687861;0.165974;0.440105;0.880075;0.829201;0.330337;0.228968;0.893372;0.35036;0.68667;0.956468;0.58864;0.657304;0.858676;0.43956;0.92397;0.398437;0.814767;0.684219;0.910972;0.482491;0.215825;0.950252;0.920128;0.14766;0.881062;0.641081;0.431953;0.619596;0.281059;0.786002;0.307458;0.447034;0.226107;0.187533;0.276235;0.556444;0.416501;0.169607;0.906804;0.103171;0.126075;0.495444;0.760475;0.984752;0.935004;0.684445;0.383188;0.749771;0.368664;0.29416;0.232262;0.584489;0.244413;0.15239;0.732149;0.125475;0.79347;0.164102;0.745071;0.0745298;0.950104;0.0525293;0.521563;0.176211;0.240062;0.797798;0.732654;0.656564;0.967405;0.639458;0.759735;0.0934805;0.134902;0.52021;0.0782321;0.0699064;0.204655;0.46142;0.819677;0.573319;0.755581;0.0519388;0.157807;0.999994;0.204329;0.889956;0.125468;0.997799;0.0540576;0.87054;0.0723288;0.00416161;0.923069;0.593892;0.180372;0.163132;0.39169;0.913027;0.819695;0.359095;0.552485;0.57943;0.452576;0.687387;0.0996401;0.530808;0.757294;0.304295;0.992228;0.576971;0.877614;0.747809;0.62891;0.0354209;0.747803;0.833239;0.925377;0.873271;0.831038;0.979434;0.743811;0.903366;0.983596;0.66688;0.497259;0.163968;0.830012;0.888949;0.0769947;0.649707;0.248044;0.62948;0.229137;0.70062;0.316867;0.328777;0.231428;0.074161;0.633072;0.223656;0.651132;0.510686;0.971466;0.280042;0.546107;0.719269;0.113281;0.471483;0.59254;0.944318;0.450918;0.336351;0.847684;0.434513;0.00323146;0.344943;0.598481;0.833243;0.233892;0.675476;0.48295;0.481936;0.304956;0.712087;0.182556;0.621823;0.0408643;0.413984;0.695984;0.673936;0.63764;0.347116;0.184622;0.609106;0.627158;0.730729;0.328374;0.740438;0.202213;0.920914;0.684757;0.65313;0.257265;0.532441;0.0876436;0.260497;0.877384;0.686125;0.0937402;0.111276;0.361601;0.57669;0.593211;0.666557;0.288778;0.775767;0.288379;0.329642;0.189751;0.984363;0.00357857;0.827391;0.331479;0.188201;0.436497;0.958637;0.91893;0.764871;0.699075;0.121143;0.685786;0.383832;0.774274;0.943051;0.916273;0.861917;0.203548;0.793657;0.548042;0.297288;0.904932;0.909643;0.873979;0.498144;0.5762;0.162757;0.273911;0.864579;0.492399;0.463662;0.848942;0.495977;0.291053;0.180421;0.684178;0.72755;0.139058;0.603109;0.492422;0.838134;0.724252;0.178208;0.221966;0.498525;0.121259;0.138238;0.360443;0.324807;0.931895;0.908485;0.622095;0.836828;0.818128;0.496074;0.334972;0.394327;0.658831;0.608883;0.258906;0.15123;0.072545;0.107848;0.647207;0.363598;0.28827;0.331386;0.0911486;0.427328;0.934495;0.58357;0.265461;0.658747;0.761778;0.487427;0.157272;0.883037;0.625665;0.517715;0.207844;0.557561;0.426199;0.829939;0.394388;0.244327;0.326013;0.72936;0.638654;0.984845;0.338243;0.89756;0.136075;0.410788;0.00540855;0.783282;0.774386;0.293678;0.114668;0.865535;0.721006;0.0491625;0.449105;0.986467;0.707909;0.210883;0.473894;0.865181;0.0939195;0.0995593;0.382896;0.301763;0.65712;0.809095;0.131702;0.0515083;0.0534223;0.457716;0.780868;0.692076;0.44256;0.119111;0.589637;0.578635;0.529899;0.595045;0.361917;0.304285;0.888723;0.476585;0.16982;0.609729;0.525747;0.618925;0.596196;0.233656;0.829808;0.0700902;0.0988374;0.923728;0.169649;0.481733;0.225491;0.826769;0.290829;0.357193;0.878278;0.344251;0.814909;0.659146;0.0363274;0.257469;0.778257;0.625964;0.836104;0.308157;0.221009;0.198021;0.612442;0.109733;0.674605;0.782262;0.719462;0.200352;0.401188;0.315658;0.434009;0.230996;0.385748;0.532846;0.154724;0.555398;0.0145793;0.380215;0.382167;0.305408;0.737408;0.260445;0.649659;0.552316;0.919591;0.685986;0.809785;0.697848;0.31195;0.645889;0.00600477;0.53296;0.84391;0.618447;0.642693;0.518515;0.400709;0.362154;0.718867;0.801897;0.677812;0.152876;0.0328927;0.0635606;0.685722;0.187616;0.618958;0.700301;0.567831;0.00112548;0.00570914;0.305239;0.26157;0.655368;0.857555;0.181161;0.341354;0.667341;0.879009;0.653305;0.31323;0.885014;0.186265;0.157139;0.503461;0.828957;0.675654;0.90417;0.191112;0.394521;0.706067;0.868924;0.547397;0.738959;0.932485;0.233119;0.926576;0.551443;0.93342;0.494407;0.552568;0.939129;0.799646;0.814139;0.594497;0.657201;0.9953;0.935852;0.324541;0.874309;0.589157;0.637771;0.759324;0.775421;0.79491;0.262785;0.604379;0.470564;0.166955;0.79549;0.865085;0.873021;0.664414;0.412483;0.611981;0.596899;0.645602;0.538557;0.148342;0.579022;0.0329634;0.70091;0.518151;0.832609;0.515049;0.112648;0.48981;0.510349;0.0484997;0.814351;0.384658;0.637656;0.452122;0.143982;0.413078;0.247033;0.406767;0.0174566;0.717597;0.573721;0.812947;0.582682;0.446743;0.477361;0.995165;0.0587232;0.0742604;0.640766;0.59728;0.222602;0.219788;0.630243;0.923513;0.737939;0.462852;0.438562;0.850586;0.952662;0.948911;0.899086;0.767014;0.333569;0.536743;0.219136;0.477551;0.94982;0.466169;0.884318;0.967277;0.183765;0.458039;0.780224;0.766448;0.904782;0.257585;0.761612;0.963505;0.331846;0.402379;0.560785;0.554448;0.622167;0.191028;0.477961;0.360105;0.65388;0.916523;0.210692;0.606542;0.865434;0.109778;0.373556;0.199003;0.64652;0.592692;0.676554;0.596341;0.0588605;0.560872;0.563617;0.242626;0.0189108;0.343841;0.00907344;0.923692;0.601427;0.770686;0.887197;0.933273;0.173065;0.447982;0.487721;0.795231;0.639009;0.965682;0.155336;0.292889;0.882204;0.366028;0.899431;0.747638;0.475806;0.272987;0.94664;0.122326;0.865679;0.623194;0.718666;0.92454;0.184066;0.282284;0.167165;0.202977;0.626125;0.176239;0.126669;0.227552;0.946925;0.0138663;0.160824;0.119989;0.461848;0.648545;0.915221;0.100857;0.614227;0.070557;0.393746;0.496431;0.436585;0.293177;0.244069;0.912391;0.566164;0.190709;0.0347164;0.431844;0.813904;0.753383;0.356383;0.99797;0.0356664;0.523548;0.200947;0.661792;0.699787;0.327616;0.889343;0.646712;0.341482;0.0501679;0.766701;0.80333;0.698713;0.681922;0.904187;0.31294;0.752479;0.297933;0.809371;0.189064;0.591111;0.0534394;0.101454;0.157275;0.244149;0.136171;0.589119;0.0580523;0.889553;0.945502;0.0560222;0.92522;0.46905;0.256969;0.587011;0.168837;0.584585;0.476355;0.815549;0.926068;0.526523;0.58225;0.729398;0.225236;0.264172;0.633585;0.538175;0.0166506;0.931518;0.347546;0.205714;0.522629;0.400985;0.307168;0.679904;0.645134;0.443339;0.269022;0.703186;0.332892;0.214524;0.759208;0.258112;0.683574;0.0161775;0.845123;0.852411;0.600763;0.321478;0.66796;0.52683;0.848;0.25021;0.256228;0.0732357;0.514382;0.889813;0.611411;0.531033;0.821331;0.958957;0.736747;0.343959;0.359942;0.0439153;0.0238632;0.0050762;0.487254;0.292886;0.708262;0.820146;0.50741;0.467471;0.0782579;0.190984;0.483648;0.923381;0.0433947;0.084411;0.244858;0.711355;0.611241;0.0928584;0.961565;0.867469;0.166094;0.475947;0.757282;0.777505;0.00698012;0.578613;0.736462;0.743727;0.922572;0.0964041;0.787642;0.946435;0.10148;0.274897;0.239321;0.809743;0.0950428;0.74673;0.277214;0.173301;0.937714;0.760862;0.0966814;0.981109;0.845273;0.34154;0.692463;0.456514;0.434398;0.654029;0.323983;0.600492;0.129976;0.081265;0.377997;0.136956;0.659878;0.114459;0.880683;0.58245;0.210863;0.668326;0.528885;0.312343;0.943222;0.768206;0.122086;0.0382648;0.514936;0.3993;0.211565;0.45265;0.160162;0.308247;0.433758;0.00543489;0.649787;0.126222;0.461949;0.0841846;0.78025;0.785932;0.684677;0.910227;0.867197;0.0626739;0.0471826;0.527075;0.177133;0.927866;0.109525;0.387996;0.596191;0.638409;0.70034;0.539413;0.406615;0.822426;0.577678;0.921551;0.221726;0.789244;0.374201;0.381888;0.0974906;0.807959;0.387323;0.747277;0.934181;0.849272;0.831462;0.714432;0.635204;0.516139;0.624658;0.502401;0.578813;0.671841;0.0294762;0.755946;0.599707;0.139001;0.143942;0.195898;0.77741;0.844281;0.735311;0.184025;0.666707;0.31299;0.105576;0.888433;0.102233;0.479777;0.270321;0.199724;0.287736;0.657643;0.947001;0.221918;0.506915;0.778463;0.936349;0.142119;0.294601;0.561007;0.64452;0.873414;0.232848;0.673996;0.629359;0.832555;0.812997;0.773301;0.0284525;0.590407;0.617582;0.763764;0.774432;0.284289;0.0767534;0.880009;0.172722;0.178987;0.359786;0.443043;0.37871;0.647522;0.100686;0.325711;0.86944;0.6076;0.104174;0.805789;0.749719;0.398775;0.366796;0.394239;0.272189;0.599644;0.0682348;0.901549;0.432199;0.881232;0.67485;0.460652;0.471639;0.292432;0.224415;0.246071;0.576721;0.301169;0.12608;0.749443;0.480155;0.485866;0.192486;0.858866;0.133388;0.293171;0.184577;0.00282779;0.900772;0.288752;0.808617;0.650491;0.687527;0.175413;0.0447295;0.959716;0.775058;0.112964;0.861265;0.207257;0.994196;0.536115;0.667908;0.465835;0.828546;0.892324;0.711906;0.405267;0.193493;0.837986;0.154711;0.673648;0.323852;0.347196;0.532514;0.45724;0.640368;0.717092;0.460067;0.54114;0.00584319;0.268684;0.19163;0.69337;0.444097;0.23636;0.653087;0.219155;0.349324;0.514352;0.426412;0.34352;0.0504663;0.0943199;0.809355;0.879013;0.986644;0.521261;0.28428 diff --git a/tests/csv_samples/connection.csv b/tests/csv_samples/connection.csv new file mode 100644 index 00000000..571d2210 --- /dev/null +++ b/tests/csv_samples/connection.csv @@ -0,0 +1,5 @@ +ConnectionClass,Person@id,Person@id,prop_type +BloodRelation,1,2,brother +BloodRelation,14,16,sister +BloodRelation,14,15,mother +BloodRelation,14,13,father \ No newline at end of file diff --git a/tests/csv_samples/person.csv b/tests/csv_samples/person.csv new file mode 100644 index 00000000..2ab3a370 --- /dev/null +++ b/tests/csv_samples/person.csv @@ -0,0 +1,6 @@ +EntityClass,prop_name,prop_lastname,prop_id,prop_age +Person,Ali,Hum,1,2 +Person,Hamzah,Hom,2,3 +Person,Tala,Ali,16,45 +Person,Soha,Khalid,14,12 +Person,Shah,Hum,15,40 diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 4f3df5e6..2ee2f92e 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -1,5 +1,5 @@ sh cleandbs.sh - +mkdir test_db_client mkdir dbs # necessary for Descriptors mkdir temp # necessary for Videos mkdir videos_tests @@ -17,4 +17,3 @@ echo 'Running C++ tests...' --gtest_filter=-ImageTest.CreateNameTDB:ImageTest.NoMetadata:VideoTest.CreateUnique:Descriptors_Add.add_1by1_and_search_1k # kill -9 $cpp_unittest_pid $client_test_pid -# sh cleandbs.sh diff --git a/tests/unit_tests/client_blob.cc b/tests/unit_tests/client_blob.cc index 190478ce..7af5259d 100644 --- a/tests/unit_tests/client_blob.cc +++ b/tests/unit_tests/client_blob.cc @@ -1,10 +1,19 @@ #include "meta_data_helper.h" +#include "CSVParserUtil.h" TEST(BLOB, add_Blob){ std::string filename ="../tests/test_images/large1.jpg"; std::vector blobs; - + VDMS::CSVParserUtil csv_util; + std::string* blob_data_ptr = nullptr; + + csv_util.read_blob_image(filename, &blob_data_ptr); + + if(blob_data_ptr!=nullptr){ + blobs.push_back(blob_data_ptr); + // std::cout <<*blobs[0] <read_blob(filename)); + // -blobs.push_back(meta_obj->read_blob(filename)); meta_obj->_aclient.reset ( new VDMS::VDMSClient(meta_obj->get_server(), meta_obj->get_port())); Json::Value tuple ; tuple=meta_obj->construct_Blob(); diff --git a/tests/unit_tests/client_csv.cc b/tests/unit_tests/client_csv.cc new file mode 100644 index 00000000..fa1a7812 --- /dev/null +++ b/tests/unit_tests/client_csv.cc @@ -0,0 +1,238 @@ +#include "meta_data_helper.h" +#include "CSVParser.h" +TEST(CLIENT_CPP_CSV, parse_csv_entity) +{ + + std::string filename = "../tests/csv_samples/CSVformat100.csv"; + size_t num_threads = 5; + std::string vdms_server = "localhost"; + int port = 55558; + std::vector all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["AddEntity"]["status"].asInt(), 0); + } +} + +// TEST(CLIENT_CPP_CSV, parse_update_csv_entity) +// { + +// std::string filename = "../tests/csv_samples/update_entity.csv"; +// size_t num_threads = 2; +// std::string vdms_server ="localhost"; +// int port = 55558; +// std::vector all_results; +// VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + +// all_results = csv_parser.parse(); +// Json::Value result; +// Json::Reader _reader; +// for (int k=0; k all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["AddConnection"]["status"].asInt(), 0); + } +} +TEST(CLIENT_CPP_CSV, parse_csv_images) +{ + std::string filename = "../tests/csv_samples/Image.csv"; + size_t num_threads = 5; + std::string vdms_server = "localhost"; + int port = 55558; + std::vector all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["AddImage"]["status"].asInt(), 0); + } +} + +TEST(CLIENT_CPP_CSV, parse_csv_descriptor_set) +{ + std::string filename = "../tests/csv_samples/DescriptorSet.csv"; + size_t num_threads = 5; + std::string vdms_server = "localhost"; + int port = 55558; + + std::vector all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["AddDescriptorSet"]["status"].asInt(), 0); + } +} + +TEST(CLIENT_CPP_CSV, parse_csv_descriptor) +{ + std::string filename = "../tests/csv_samples/Descriptor.csv"; + size_t num_threads = 5; + std::string vdms_server = "localhost"; + int port = 55558; + + std::vector all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["AddDescriptor"]["status"].asInt(), 0); + } +} +TEST(CLIENT_CPP_CSV, parse_csv_bb) +{ + std::string filename = "../tests/csv_samples/Rectangle.csv"; + size_t num_threads = 5; + std::string vdms_server = "localhost"; + int port = 55558; + + std::vector all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["AddBoundingBox"]["status"].asInt(), 0); + } +} +TEST(CLIENT_CPP_CSV, parse_csv_video) +{ + std::string filename = "../tests/csv_samples/Video.csv"; + size_t num_threads = 5; + std::string vdms_server = "localhost"; + int port = 55558; + std::vector all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["AddVideo"]["status"].asInt(), 0); + } +} + +TEST(CLIENT_CPP_CSV, parse_csv_invalid_entity) +{ + std::string filename = "../tests/csv_samples/invalid.csv"; + std::ofstream csv_file; + csv_file.open(filename); + csv_file << "EntityInvalidTest,prop_name,prop_lastname,prop_id,prop_age\n"; + csv_file << "Person,Ali,Hum,1,2\n"; + csv_file.close(); + + size_t num_threads = 1; + std::string vdms_server = "localhost"; + int port = 55558; + std::vector all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + remove(filename.c_str()); + + Json::Value result; + Json::Reader _reader; + _reader.parse(all_results[0].json.c_str(), result); + EXPECT_EQ(result["status"].asInt(), -1); + EXPECT_EQ(result["info"].asString(), "Command does not exist"); +} + +TEST(CLIENT_CPP_CSV, parse_csv_invalid_image) +{ + std::string filename = "../tests/csv_samples/invalid_file.csv"; + std::ofstream csv_file; + csv_file.open(filename); + csv_file << "ImagePath,ops_threshold,ops_crop,ops_resize,ops_flip,ops_rotate,prop_type,prop_part,format,cons_1\n"; + csv_file << "../tests/test_images/large1_invalid.jpg,350,,,,,,image1,jpg,part==image1\n"; + csv_file.close(); + + size_t num_threads = 1; + std::string vdms_server = "localhost"; + int port = 55558; + std::vector all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + remove(filename.c_str()); + + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["status"].asInt(), -1); + } +} + +TEST(CLIENT_CPP_CSV, parse_csv_invalid_video) +{ + std::string filename = "../tests/csv_samples/invalid_file.csv"; + std::ofstream csv_file; + csv_file.open(filename); + csv_file << "VideoPath,format,compressto,prop_name,ops_resize,ops_interval\n"; + csv_file << "../tests/test_videos/Megamind_invalid.avi,avi,h264,Good,,\n"; + csv_file.close(); + + size_t num_threads = 1; + std::string vdms_server = "localhost"; + int port = 55558; + std::vector all_results; + VDMS::CSVParser csv_parser(filename, num_threads, vdms_server, port); + + all_results = csv_parser.parse(); + remove(filename.c_str()); + + Json::Value result; + Json::Reader _reader; + for (int k = 0; k < all_results.size(); k++) + { + _reader.parse(all_results[k].json.c_str(), result); + EXPECT_EQ(result[k]["status"].asInt(), -1); + } +} + diff --git a/tests/unit_tests/client_videos.cc b/tests/unit_tests/client_videos.cc index 35c56a22..887ea75f 100644 --- a/tests/unit_tests/client_videos.cc +++ b/tests/unit_tests/client_videos.cc @@ -24,7 +24,7 @@ string readFileIntoString(const string& path) { -TEST(CLIENT_CPP, add_single_video){ +TEST(CLIENT_CPP_Video, add_single_video){ // std::string video; diff --git a/tests/unit_tests/meta_data_helper.h b/tests/unit_tests/meta_data_helper.h index 664fd4f0..005d7e00 100644 --- a/tests/unit_tests/meta_data_helper.h +++ b/tests/unit_tests/meta_data_helper.h @@ -50,9 +50,4 @@ class Meta_Data{ Json::Value construct_Flinng_Set(std::string&, int&); std::string get_server(){return _server_name;} int get_port() {return _port;} - - - - - -}; \ No newline at end of file +};