diff --git a/.gitignore b/.gitignore index d35fe720ad..714bd826b0 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ build/ venv/ **/.pytest_cache/ __pycache__/ +*.db3-* + diff --git a/rosbag2_compression/test/rosbag2_compression/mock_storage.hpp b/rosbag2_compression/test/rosbag2_compression/mock_storage.hpp index 3feb383d07..39df735d41 100644 --- a/rosbag2_compression/test/rosbag2_compression/mock_storage.hpp +++ b/rosbag2_compression/test/rosbag2_compression/mock_storage.hpp @@ -48,6 +48,7 @@ class MockStorage : public rosbag2_storage::storage_interfaces::ReadWriteInterfa MOCK_CONST_METHOD0(get_bagfile_size, uint64_t()); MOCK_CONST_METHOD0(get_relative_file_path, std::string()); MOCK_CONST_METHOD0(get_storage_identifier, std::string()); + MOCK_CONST_METHOD0(get_storage_extension, std::string()); MOCK_CONST_METHOD0(get_minimum_split_file_size, uint64_t()); }; diff --git a/rosbag2_cpp/CMakeLists.txt b/rosbag2_cpp/CMakeLists.txt index f93b486ac0..8e3d513923 100644 --- a/rosbag2_cpp/CMakeLists.txt +++ b/rosbag2_cpp/CMakeLists.txt @@ -1,6 +1,8 @@ cmake_minimum_required(VERSION 3.5) project(rosbag2_cpp) +add_definitions(-D_SRC_REINDEX_DIR_PATH="${CMAKE_CURRENT_SOURCE_DIR}/test/rosbag2_cpp/reindex_test_bags") + # Default to C99 if(NOT CMAKE_C_STANDARD) set(CMAKE_C_STANDARD 99) @@ -61,9 +63,11 @@ add_library(${PROJECT_NAME} SHARED src/rosbag2_cpp/typesupport_helpers.cpp src/rosbag2_cpp/types/introspection_message.cpp src/rosbag2_cpp/writer.cpp - src/rosbag2_cpp/writers/sequential_writer.cpp) + src/rosbag2_cpp/writers/sequential_writer.cpp + src/rosbag2_cpp/reindexer.cpp) ament_target_dependencies(${PROJECT_NAME} + PUBLIC ament_index_cpp pluginlib rclcpp @@ -188,6 +192,7 @@ if(BUILD_TESTING) ament_target_dependencies(test_message_cache rosbag2_storage) endif() + # If compiling with gcc, run this test with sanitizers enabled ament_add_gmock(test_ros2_message test/rosbag2_cpp/types/test_ros2_message.cpp diff --git a/rosbag2_cpp/include/rosbag2_cpp/reindexer.hpp b/rosbag2_cpp/include/rosbag2_cpp/reindexer.hpp new file mode 100644 index 0000000000..5dd19c2170 --- /dev/null +++ b/rosbag2_cpp/include/rosbag2_cpp/reindexer.hpp @@ -0,0 +1,127 @@ +// Copyright 2020 DCS Corporation, All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// DISTRIBUTION A. Approved for public release; distribution unlimited. +// OPSEC #4584. +// +// Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS +// Part 252.227-7013 or 7014 (Feb 2014). +// +// This notice must appear in all copies of this file and its derivatives. + +#ifndef ROSBAG2_CPP__REINDEXER_HPP_ +#define ROSBAG2_CPP__REINDEXER_HPP_ + +#include +#include +#include +#include + +#include "rcpputils/filesystem_helper.hpp" + +#include "rosbag2_cpp/converter.hpp" +#include "rosbag2_cpp/reader.hpp" +#include "rosbag2_cpp/serialization_format_converter_factory.hpp" +#include "rosbag2_cpp/serialization_format_converter_factory_interface.hpp" +#include "rosbag2_cpp/visibility_control.hpp" + +#include "rosbag2_storage/metadata_io.hpp" +#include "rosbag2_storage/storage_factory.hpp" +#include "rosbag2_storage/storage_factory_interface.hpp" +#include "rosbag2_storage/storage_options.hpp" +#include "rosbag2_storage/storage_filter.hpp" +#include "rosbag2_storage/storage_interfaces/read_only_interface.hpp" + +// This is necessary because of using stl types here. It is completely safe, because +// a) the member is not accessible from the outside +// b) there are no inline functions. +#ifdef _WIN32 +# pragma warning(push) +# pragma warning(disable:4251) +#endif + +namespace rosbag2_cpp +{ + +/** + * Tool to reconstruct bag metadata files in the event of loss or corruption + * + * Reindexing is an operation where a bag that is missing a metadata.yaml file can have a new + * file created through parsing of the metadata stored within the actual files of the bag. + * For instance: Imagine we are working with SQL databases (.db3). We can open the individual + * .db3 files within the bag and read their metadata (not the messages themselves) to replicate + * a usable metadata.yaml file, so that the bag can once again be read by the standard read + * command. + * + * Reindexing has some limitations - It cannot perfectly replicate the original metadata file, + * since some information known by the program from the start up command cannot be found + * within the metadata. But it should at least repair a bag to the point it can be read + * again. + * + */ +class ROSBAG2_CPP_PUBLIC Reindexer +{ +public: + Reindexer( + std::unique_ptr storage_factory = + std::make_unique(), + std::unique_ptr metadata_io = + std::make_unique()); + + virtual ~Reindexer() = default; + + /// Use the supplied storage options to reindex a bag defined by the storage options URI. + /* + * \param storage_options Provides best-guess parameters for the bag's original settings. + */ + void reindex(const rosbag2_storage::StorageOptions & storage_options); + +protected: + std::unique_ptr storage_factory_{}; + std::unique_ptr metadata_io_{}; + rosbag2_storage::BagMetadata metadata_{}; + std::vector topics_metadata_{}; + +private: + std::string regex_bag_pattern_; + rcpputils::fs::path base_folder_; // The folder that the bag files are in + std::shared_ptr converter_factory_{}; + void get_bag_files( + const rcpputils::fs::path & base_folder, + std::vector & output); + + // Prepares the metadata by setting initial values. + void init_metadata( + const std::vector & files, + const rosbag2_storage::StorageOptions & storage_options); + + // Attempts to harvest metadata from all bag files, and aggregates the result + void aggregate_metadata( + const std::vector & files, + const std::unique_ptr & bag_reader, + const rosbag2_storage::StorageOptions & storage_options); + + // Comparison function for std::sort with our filepath convention + bool compare_relative_file( + const rcpputils::fs::path & first_path, + const rcpputils::fs::path & second_path); +}; + +} // namespace rosbag2_cpp + +#ifdef _WIN32 +# pragma warning(pop) +#endif + +#endif // ROSBAG2_CPP__REINDEXER_HPP_ diff --git a/rosbag2_cpp/src/rosbag2_cpp/reindexer.cpp b/rosbag2_cpp/src/rosbag2_cpp/reindexer.cpp new file mode 100644 index 0000000000..c33ed152dd --- /dev/null +++ b/rosbag2_cpp/src/rosbag2_cpp/reindexer.cpp @@ -0,0 +1,265 @@ +// Copyright 2020 DCS Corporation, All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// DISTRIBUTION A. Approved for public release; distribution unlimited. +// OPSEC #4584. +// +// Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS +// Part 252.227-7013 or 7014 (Feb 2014). +// +// This notice must appear in all copies of this file and its derivatives. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rcpputils/asserts.hpp" +#include "rcpputils/filesystem_helper.hpp" + +#include "rcutils/filesystem.h" + +#include "rosbag2_cpp/logging.hpp" +#include "rosbag2_cpp/reader.hpp" +#include "rosbag2_cpp/reindexer.hpp" + +#include "rosbag2_storage/storage_options.hpp" + +namespace rosbag2_cpp +{ +Reindexer::Reindexer( + std::unique_ptr storage_factory, + std::unique_ptr metadata_io) +: storage_factory_(std::move(storage_factory)), + metadata_io_(std::move(metadata_io)) +{ + regex_bag_pattern_ = R"(.+_(\d+)\.([a-zA-Z0-9])+)"; +} + +/// Determine which path should be placed first in a vector ordered by file number. +/** + * Used to re-order discovered bag files, since the filesystem discovery functions + * don't guarantee a preserved order + */ +bool Reindexer::compare_relative_file( + const rcpputils::fs::path & first_path, + const rcpputils::fs::path & second_path) +{ + std::regex regex_rule(regex_bag_pattern_, std::regex_constants::ECMAScript); + + std::smatch first_match; + std::smatch second_match; + + auto first_path_string = first_path.string(); + auto second_path_string = second_path.string(); + + auto first_regex_good = std::regex_match(first_path_string, first_match, regex_rule); + auto second_regex_good = std::regex_match(second_path_string, second_match, regex_rule); + + if (!first_regex_good) { + std::stringstream ss; + ss << "Path " << first_path.string() << + "didn't meet expected naming convention: " << regex_bag_pattern_; + std::string error_text = ss.str(); + throw std::runtime_error(error_text.c_str()); + } else if (!second_regex_good) { + std::stringstream ss; + ss << "Path " << second_path.string() << + "didn't meet expected naming convention: " << regex_bag_pattern_; + std::string error_text = ss.str(); + throw std::runtime_error(error_text.c_str()); + } + + auto first_db_num = std::stoul(first_match.str(1), nullptr, 10); + auto second_db_num = std::stoul(second_match.str(1), nullptr, 10); + + return first_db_num < second_db_num; +} + +/// Retrieve bag storage files from the bag directory. +/** + * @param base_folder: The bag directory that contains all the bag storage files + * @param output: A vector to save the discovered files inside of + * The files will be `emplace_back`-ed on the passed vector + */ +void Reindexer::get_bag_files( + const rcpputils::fs::path & base_folder, + std::vector & output) +{ + std::regex regex_rule(regex_bag_pattern_, std::regex_constants::ECMAScript); + auto allocator = rcutils_get_default_allocator(); + auto dir_iter = rcutils_dir_iter_start(base_folder.string().c_str(), allocator); + + // Make sure there are files in the directory + if (dir_iter == nullptr) { + throw std::runtime_error("Empty directory."); + } + + // Get all file names in directory + do { + auto found_file = rcpputils::fs::path(dir_iter->entry_name); + ROSBAG2_CPP_LOG_DEBUG_STREAM("Found file: " << found_file.string()); + + if (std::regex_match(found_file.string(), regex_rule)) { + auto full_path = base_folder / found_file; + output.emplace_back(full_path); + } + } while (rcutils_dir_iter_next(dir_iter)); + + // Sort relative file path by database number + std::sort( + output.begin(), output.end(), + [&, this](rcpputils::fs::path a, rcpputils::fs::path b) { + return compare_relative_file(a, b); + }); +} + +/// Prepare a fresh BagMetadata object for reindexing. +/** + * Creates a new `BagMetadata` object with the `storage_identifier` and `relative_file_paths` filled in + * Also fills in `starting_time` with a dummy default value. Important for later functions + */ +void Reindexer::init_metadata( + const std::vector & files, + const rosbag2_storage::StorageOptions & storage_options) +{ + metadata_ = rosbag2_storage::BagMetadata{}; + + metadata_.storage_identifier = storage_options.storage_id; + metadata_.starting_time = std::chrono::time_point( + std::chrono::nanoseconds::max()); + + // Record the relative paths to the metadata + for (const auto & path : files) { + auto cleaned_path = path.filename().string(); + metadata_.relative_file_paths.push_back(cleaned_path); + } +} + +/// Iterate through the bag files to collect various metadata parameters +/** + * Collects the topic metadata, `starting_time`, and `duration` portions of the `BagMetadata` + * being constructed + * @param: files The list of bag files to reindex + * @param: storage_options Used to construct the `Reader` needed to parse the bag files + */ +void Reindexer::aggregate_metadata( + const std::vector & files, + const std::unique_ptr & bag_reader, + const rosbag2_storage::StorageOptions & storage_options) +{ + std::map temp_topic_info; + + // In order to most accurately reconstruct the metadata, we need to + // visit each of the contained relative files files in the bag, + // open them, read the info, and write it into an aggregated metadata object. + ROSBAG2_CPP_LOG_DEBUG_STREAM("Extracting metadata from database(s)"); + for (const auto & f_ : files) { + ROSBAG2_CPP_LOG_DEBUG_STREAM("Extracting from file: " + f_.string()); + + metadata_.bag_size += f_.file_size(); + + // Set up reader + rosbag2_storage::StorageOptions temp_so = { + f_.string(), + storage_options.storage_id, + storage_options.max_bagfile_size, + storage_options.max_bagfile_duration, + storage_options.max_cache_size, + storage_options.storage_config_uri + }; + + // We aren't actually interested in reading messages, so use a blank converter option + rosbag2_cpp::ConverterOptions blank_converter_options {}; + bag_reader->open(temp_so, blank_converter_options); + auto temp_metadata = bag_reader->get_metadata(); + + if (temp_metadata.starting_time < metadata_.starting_time) { + metadata_.starting_time = temp_metadata.starting_time; + } + metadata_.duration += temp_metadata.duration; + ROSBAG2_CPP_LOG_DEBUG_STREAM("New duration: " + std::to_string(metadata_.duration.count())); + metadata_.message_count += temp_metadata.message_count; + + // Add the topic metadata + for (const auto & topic : temp_metadata.topics_with_message_count) { + auto found_topic = temp_topic_info.find(topic.topic_metadata.name); + if (found_topic == temp_topic_info.end()) { + // It's a new topic. Add it. + temp_topic_info[topic.topic_metadata.name] = topic; + } else { + ROSBAG2_CPP_LOG_DEBUG_STREAM("Found topic!"); + // Merge in the new information + found_topic->second.message_count += topic.message_count; + if (topic.topic_metadata.offered_qos_profiles != "") { + found_topic->second.topic_metadata.offered_qos_profiles = + topic.topic_metadata.offered_qos_profiles; + } + if (topic.topic_metadata.serialization_format != "") { + found_topic->second.topic_metadata.serialization_format = + topic.topic_metadata.serialization_format; + } + if (topic.topic_metadata.type != "") { + found_topic->second.topic_metadata.type = topic.topic_metadata.type; + } + } + } + + bag_reader->reset(); + } + + // Convert the topic map into topic metadata + for (auto & topic : temp_topic_info) { + metadata_.topics_with_message_count.emplace_back(topic.second); + } +} + +/// Reconstruct a bag's `metadata.yaml` file from the enclosed bag files. +/** + * The reindexer opens the files within the bag directory and uses the metadata of the files to + * reconstruct the metadata file. Currently does not support compressed bags. + * @param: storage_options The best-guess original storage options for the bag + */ +void Reindexer::reindex(const rosbag2_storage::StorageOptions & storage_options) +{ + base_folder_ = storage_options.uri; + ROSBAG2_CPP_LOG_INFO_STREAM("Beginning reindexing bag in directory: " << base_folder_); + + auto metadata_io_default = std::make_unique(); + auto bag_reader = std::make_unique( + std::move(storage_factory_), converter_factory_, std::move(metadata_io_default)); + + // Identify all bag files + std::vector files; + get_bag_files(base_folder_, files); + if (files.empty()) { + throw std::runtime_error("No database files found for reindexing. Abort"); + } + + init_metadata(files, storage_options); + ROSBAG2_CPP_LOG_DEBUG_STREAM("Completed init_metadata"); + + // Collect all metadata from database files + aggregate_metadata(files, bag_reader, storage_options); + ROSBAG2_CPP_LOG_DEBUG_STREAM("Completed aggregate_metadata"); + + metadata_io_->write_metadata(base_folder_.string(), metadata_); + ROSBAG2_CPP_LOG_INFO("Reindexing complete."); +} +} // namespace rosbag2_cpp diff --git a/rosbag2_cpp/test/rosbag2_cpp/mock_storage.hpp b/rosbag2_cpp/test/rosbag2_cpp/mock_storage.hpp index 84bbf47ac3..303ddf9907 100644 --- a/rosbag2_cpp/test/rosbag2_cpp/mock_storage.hpp +++ b/rosbag2_cpp/test/rosbag2_cpp/mock_storage.hpp @@ -49,6 +49,7 @@ class MockStorage : public rosbag2_storage::storage_interfaces::ReadWriteInterfa MOCK_CONST_METHOD0(get_bagfile_size, uint64_t()); MOCK_CONST_METHOD0(get_relative_file_path, std::string()); MOCK_CONST_METHOD0(get_storage_identifier, std::string()); + MOCK_CONST_METHOD0(get_storage_extension, std::string()); MOCK_CONST_METHOD0(get_minimum_split_file_size, uint64_t()); }; diff --git a/rosbag2_storage/test/rosbag2_storage/test_constants.hpp b/rosbag2_storage/test/rosbag2_storage/test_constants.hpp index d7a2ebc3ef..af91e80dd3 100644 --- a/rosbag2_storage/test/rosbag2_storage/test_constants.hpp +++ b/rosbag2_storage/test/rosbag2_storage/test_constants.hpp @@ -18,6 +18,8 @@ namespace test_constants { +constexpr const char * const READ_WRITE_PLUGIN_EXTENSION = ".rwplugin"; +constexpr const char * const READ_ONLY_PLUGIN_EXTENSION = ".roplugin"; constexpr const char * const READ_WRITE_PLUGIN_IDENTIFIER = "ReadWritePlugin"; constexpr const char * const READ_ONLY_PLUGIN_IDENTIFIER = "ReadOnlyPlugin"; constexpr const char * const DUMMY_FILEPATH = "/path/to/storage"; diff --git a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp index 1af8a405ce..20063af55a 100644 --- a/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp +++ b/rosbag2_storage_default_plugins/src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp @@ -440,19 +440,19 @@ rosbag2_storage::BagMetadata SqliteStorage::get_metadata() auto statement = database_->prepare_statement( "SELECT name, type, serialization_format, COUNT(messages.id), MIN(messages.timestamp), " - "MAX(messages.timestamp) " + "MAX(messages.timestamp), offered_qos_profiles " "FROM messages JOIN topics on topics.id = messages.topic_id " "GROUP BY topics.name;"); auto query_results = statement->execute_query< std::string, std::string, std::string, int, rcutils_time_point_value_t, - rcutils_time_point_value_t>(); + rcutils_time_point_value_t, std::string>(); rcutils_time_point_value_t min_time = INT64_MAX; rcutils_time_point_value_t max_time = 0; for (auto result : query_results) { metadata.topics_with_message_count.push_back( { - {std::get<0>(result), std::get<1>(result), std::get<2>(result), ""}, + {std::get<0>(result), std::get<1>(result), std::get<2>(result), std::get<6>(result)}, static_cast(std::get<3>(result)) }); diff --git a/rosbag2_tests/CMakeLists.txt b/rosbag2_tests/CMakeLists.txt index 3f31e8e378..5530884f8e 100644 --- a/rosbag2_tests/CMakeLists.txt +++ b/rosbag2_tests/CMakeLists.txt @@ -97,6 +97,18 @@ if(BUILD_TESTING) rosbag2_storage) endif() + ament_add_gmock(test_reindex + test/rosbag2_tests/test_reindexer.cpp) + if(TARGET test_reindex) + ament_target_dependencies(test_reindex + rclcpp + rosbag2_cpp + rosbag2_storage + rosbag2_storage_default_plugins + rosbag2_test_common) + endif() + + ament_add_gmock(test_rosbag2_cpp_api test/rosbag2_tests/test_rosbag2_cpp_api.cpp WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_0.db3.zstd b/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_0.db3.zstd new file mode 100644 index 0000000000..47efd7e928 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_0.db3.zstd differ diff --git a/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_1.db3.zstd b/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_1.db3.zstd new file mode 100644 index 0000000000..46a9604a27 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_1.db3.zstd differ diff --git a/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_2.db3.zstd b/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_2.db3.zstd new file mode 100644 index 0000000000..f901849587 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_2.db3.zstd differ diff --git a/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_3.db3.zstd b/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_3.db3.zstd new file mode 100644 index 0000000000..53c962965d Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/file_compression/file_compression_3.db3.zstd differ diff --git a/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_0.db3 b/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_0.db3 new file mode 100644 index 0000000000..cb496d68a3 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_0.db3 differ diff --git a/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_1.db3 b/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_1.db3 new file mode 100644 index 0000000000..fe3aba22e7 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_1.db3 differ diff --git a/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_2.db3 b/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_2.db3 new file mode 100644 index 0000000000..11a87eb30c Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_2.db3 differ diff --git a/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_3.db3 b/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_3.db3 new file mode 100644 index 0000000000..9b189dfa13 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/message_compression/message_compression_3.db3 differ diff --git a/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_0.db3 b/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_0.db3 new file mode 100644 index 0000000000..3edf751223 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_0.db3 differ diff --git a/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_1.db3 b/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_1.db3 new file mode 100644 index 0000000000..f367b03ac9 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_1.db3 differ diff --git a/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_2.db3 b/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_2.db3 new file mode 100644 index 0000000000..81735ad374 Binary files /dev/null and b/rosbag2_tests/resources/reindex_test_bags/multiple_files/multiple_files_2.db3 differ diff --git a/rosbag2_tests/resources/reindex_test_bags/target_metadata/file_compression/metadata.yaml b/rosbag2_tests/resources/reindex_test_bags/target_metadata/file_compression/metadata.yaml new file mode 100644 index 0000000000..764e22a4c1 --- /dev/null +++ b/rosbag2_tests/resources/reindex_test_bags/target_metadata/file_compression/metadata.yaml @@ -0,0 +1,34 @@ +rosbag2_bagfile_information: + version: 4 + storage_identifier: sqlite3 + relative_file_paths: + - file_compression/file_compression_0.db3.zstd + - file_compression/file_compression_1.db3.zstd + - file_compression/file_compression_2.db3.zstd + - file_compression/file_compression_3.db3.zstd + duration: + nanoseconds: 59997866716 + starting_time: + nanoseconds_since_epoch: 1604357089919704830 + message_count: 775 + topics_with_message_count: + - topic_metadata: + name: /topic + type: std_msgs/msg/String + serialization_format: cdr + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 2147483647\n nsec: 4294967295\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + message_count: 383 + - topic_metadata: + name: /parameter_events + type: rcl_interfaces/msg/ParameterEvent + serialization_format: cdr + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 2147483647\n nsec: 4294967295\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + message_count: 0 + - topic_metadata: + name: /rosout + type: rcl_interfaces/msg/Log + serialization_format: cdr + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 1\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 10\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + message_count: 392 + compression_format: zstd + compression_mode: FILE \ No newline at end of file diff --git a/rosbag2_tests/resources/reindex_test_bags/target_metadata/message_compression/metadata.yaml b/rosbag2_tests/resources/reindex_test_bags/target_metadata/message_compression/metadata.yaml new file mode 100644 index 0000000000..147265e2c7 --- /dev/null +++ b/rosbag2_tests/resources/reindex_test_bags/target_metadata/message_compression/metadata.yaml @@ -0,0 +1,34 @@ +rosbag2_bagfile_information: + version: 4 + storage_identifier: sqlite3 + relative_file_paths: + - message_compression/message_compression_0.db3 + - message_compression/message_compression_1.db3 + - message_compression/message_compression_2.db3 + - message_compression/message_compression_3.db3 + duration: + nanoseconds: 59872587605 + starting_time: + nanoseconds_since_epoch: 1604581538133709023 + message_count: 1127 + topics_with_message_count: + - topic_metadata: + name: /parameter_events + type: rcl_interfaces/msg/ParameterEvent + serialization_format: cdr + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 2147483647\n nsec: 4294967295\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false\n- history: 3\n depth: 0\n reliability: 1\n durability: 2\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 2147483647\n nsec: 4294967295\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + message_count: 0 + - topic_metadata: + name: /rosout + type: rcl_interfaces/msg/Log + serialization_format: cdr + offered_qos_profiles: "- history: 3\n depth: 0\n reliability: 1\n durability: 1\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 10\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false\n- history: 3\n depth: 0\n reliability: 1\n durability: 1\n deadline:\n sec: 2147483647\n nsec: 4294967295\n lifespan:\n sec: 10\n nsec: 0\n liveliness: 1\n liveliness_lease_duration:\n sec: 2147483647\n nsec: 4294967295\n avoid_ros_namespace_conventions: false" + message_count: 758 + - topic_metadata: + name: /topic + type: std_msgs/msg/String + serialization_format: cdr + offered_qos_profiles: "" + message_count: 369 + compression_format: zstd + compression_mode: MESSAGE \ No newline at end of file diff --git a/rosbag2_tests/resources/reindex_test_bags/target_metadata/multiple_files/metadata.yaml b/rosbag2_tests/resources/reindex_test_bags/target_metadata/multiple_files/metadata.yaml new file mode 100644 index 0000000000..6986d37388 --- /dev/null +++ b/rosbag2_tests/resources/reindex_test_bags/target_metadata/multiple_files/metadata.yaml @@ -0,0 +1,21 @@ +rosbag2_bagfile_information: + version: 4 + storage_identifier: sqlite3 + relative_file_paths: + - multiple_files_0.db3 + - multiple_files_1.db3 + - multiple_files_2.db3 + duration: + nanoseconds: 1616630880179528024 + starting_time: + nanoseconds_since_epoch: 22460796745086 + message_count: 3177 + topics_with_message_count: + - topic_metadata: + name: /chatter + type: std_msgs/msg/String + serialization_format: cdr + offered_qos_profiles: "- history: 1\n depth: 1\n reliability: 1\n durability: 2\n deadline:\n sec: 9223372036\n nsec: 854775807\n lifespan:\n sec: 9223372036\n nsec: 854775807\n liveliness: 1\n liveliness_lease_duration:\n sec: 9223372036\n nsec: 854775807\n avoid_ros_namespace_conventions: false" + message_count: 3177 + compression_format: "" + compression_mode: "" \ No newline at end of file diff --git a/rosbag2_tests/test/rosbag2_tests/test_reindexer.cpp b/rosbag2_tests/test/rosbag2_tests/test_reindexer.cpp new file mode 100644 index 0000000000..806891d722 --- /dev/null +++ b/rosbag2_tests/test/rosbag2_tests/test_reindexer.cpp @@ -0,0 +1,113 @@ +// Copyright 2021 DCS Corporation, All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// DISTRIBUTION A. Approved for public release; distribution unlimited. +// OPSEC #4584. +// +// Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS +// Part 252.227-7013 or 7014 (Feb 2014). +// +// This notice must appear in all copies of this file and its derivatives. + +#include + +#include +#include +#include +#include +#include + +#include "rcpputils/asserts.hpp" +#include "rcpputils/filesystem_helper.hpp" + +#include "rosbag2_cpp/readers/sequential_reader.hpp" +#include "rosbag2_cpp/reindexer.hpp" + +#include "rosbag2_storage/bag_metadata.hpp" +#include "rosbag2_storage/metadata_io.hpp" +#include "rosbag2_storage/topic_metadata.hpp" + +using namespace testing; // NOLINT + +class ReindexTestFixture : public Test +{ +public: + ReindexTestFixture() + { + database_path = rcpputils::fs::path(_SRC_RESOURCES_DIR_PATH) / "reindex_test_bags"; + target_dir = database_path / "target_metadata"; + } + + rcpputils::fs::path database_path; + rcpputils::fs::path target_dir; +}; + +TEST_F(ReindexTestFixture, test_multiple_files) { + auto bag_dir = database_path / "multiple_files"; + std::unique_ptr reindexer = + std::make_unique(); + + rosbag2_storage::StorageOptions so = rosbag2_storage::StorageOptions(); + so.uri = bag_dir.string(); + so.storage_id = "sqlite3"; + + reindexer->reindex(so); + + auto generated_file = rcpputils::fs::path(bag_dir) / "metadata.yaml"; + EXPECT_TRUE(generated_file.exists()); + + auto metadata_io = std::make_unique(); + auto generated_metadata = metadata_io->read_metadata(bag_dir.string()); + auto target_metadata = metadata_io->read_metadata((target_dir / "multiple_files").string()); + + EXPECT_EQ(generated_metadata.version, target_metadata.version); + + for (const auto & gen_rel_path : generated_metadata.relative_file_paths) { + EXPECT_TRUE( + std::find( + target_metadata.relative_file_paths.begin(), + target_metadata.relative_file_paths.end(), + gen_rel_path) != target_metadata.relative_file_paths.end()); + } + + // Disabled for now, since it may not be possible to 100% recreate + // original starting time from metadata + // EXPECT_EQ(generated_metadata.starting_time, target_metadata.starting_time); + + // Disabled for now, since I'm not sure how duration is created, and if it's correct (jhdcs) + // EXPECT_EQ(generated_metadata.duration, target_metadata.duration); + + EXPECT_EQ(generated_metadata.message_count, target_metadata.message_count); + + // Reindexer can only reconstruct topics that had messages, so not all topics may exist + for (const auto & gen_topic : generated_metadata.topics_with_message_count) { + EXPECT_TRUE( + std::find_if( + target_metadata.topics_with_message_count.begin(), + target_metadata.topics_with_message_count.end(), + [&gen_topic](rosbag2_storage::TopicInformation & t) { + return (t.topic_metadata.name == gen_topic.topic_metadata.name) && + (t.message_count == gen_topic.message_count) && + (t.topic_metadata.offered_qos_profiles == + gen_topic.topic_metadata.offered_qos_profiles) && + (t.topic_metadata.type == gen_topic.topic_metadata.type); + } + ) != target_metadata.topics_with_message_count.end() + ); + } + + if (generated_file.exists()) { + remove(generated_file); + } +}