From 3d4a2e5c59dddae0c379543da73466da501a6fe7 Mon Sep 17 00:00:00 2001 From: wraymo Date: Mon, 21 Oct 2024 13:46:24 -0400 Subject: [PATCH 01/17] Add a write path of the single-file archive format --- components/core/src/clp_s/ArchiveWriter.cpp | 155 ++++++++++++++++-- components/core/src/clp_s/ArchiveWriter.hpp | 56 ++++++- .../core/src/clp_s/CommandLineArguments.cpp | 4 + .../core/src/clp_s/CommandLineArguments.hpp | 3 + components/core/src/clp_s/JsonParser.cpp | 1 + components/core/src/clp_s/JsonParser.hpp | 1 + .../core/src/clp_s/SingleFileArchiveDefs.hpp | 56 +++++++ .../src/clp_s/TimestampDictionaryWriter.cpp | 4 + .../core/src/clp_s/archive_constants.hpp | 3 + components/core/src/clp_s/clp-s.cpp | 1 + 10 files changed, 267 insertions(+), 17 deletions(-) create mode 100644 components/core/src/clp_s/SingleFileArchiveDefs.hpp diff --git a/components/core/src/clp_s/ArchiveWriter.cpp b/components/core/src/clp_s/ArchiveWriter.cpp index ba540a79d..c526d5ca5 100644 --- a/components/core/src/clp_s/ArchiveWriter.cpp +++ b/components/core/src/clp_s/ArchiveWriter.cpp @@ -11,6 +11,7 @@ void ArchiveWriter::open(ArchiveWriterOption const& option) { m_id = boost::uuids::to_string(option.id); m_compression_level = option.compression_level; m_print_archive_stats = option.print_archive_stats; + m_single_file_archive = option.single_file_archive; auto archive_path = boost::filesystem::path(option.archives_dir) / m_id; boost::system::error_code boost_error_code; @@ -43,13 +44,37 @@ void ArchiveWriter::open(ArchiveWriterOption const& option) { } void ArchiveWriter::close() { - m_compressed_size += m_var_dict->close(); - m_compressed_size += m_log_dict->close(); - m_compressed_size += m_array_dict->close(); - m_compressed_size += m_timestamp_dict->close(); - m_compressed_size += m_schema_tree.store(m_archive_path, m_compression_level); - m_compressed_size += m_schema_map.store(m_archive_path, m_compression_level); - m_compressed_size += store_tables(); + auto var_dict_compressed_size = m_var_dict->close(); + auto log_dict_compressed_size = m_log_dict->close(); + auto array_dict_compressed_size = m_array_dict->close(); + auto timestamp_dict_compressed_size = m_timestamp_dict->close(); + auto schema_tree_compressed_size = m_schema_tree.store(m_archive_path, m_compression_level); + auto schema_map_compressed_size = m_schema_map.store(m_archive_path, m_compression_level); + auto [table_metadata_compressed_size, table_compressed_size] = store_tables(); + + if (m_single_file_archive) { + std::vector files{ + {constants::cArchiveSchemaTreeFile, schema_tree_compressed_size}, + {constants::cArchiveSchemaMapFile, schema_map_compressed_size}, + {constants::cArchiveVarDictFile, var_dict_compressed_size}, + {constants::cArchiveLogDictFile, log_dict_compressed_size}, + {constants::cArchiveArrayDictFile, array_dict_compressed_size}, + {constants::cArchiveTableMetadataFile, table_metadata_compressed_size}, + {constants::cArchiveTablesFile, table_compressed_size} + }; + uint64_t offset = 0; + for (auto & file : files) { + uint64_t original_size = file.o; + file.o = offset; + offset += original_size; + } + write_single_file_archive(files, timestamp_dict_compressed_size); + } else { + m_compressed_size = var_dict_compressed_size + log_dict_compressed_size + + array_dict_compressed_size + timestamp_dict_compressed_size + + schema_tree_compressed_size + schema_map_compressed_size + + table_metadata_compressed_size + table_compressed_size; + } if (m_metadata_db) { update_metadata_db(); @@ -67,6 +92,113 @@ void ArchiveWriter::close() { m_compressed_size = 0UL; } +void ArchiveWriter::write_single_file_archive( + std::vector const& files, + size_t timestamp_dict_compressed_size +) { + std::string archive_path = m_archive_path + constants::cArchiveFile; + FileWriter archive_writer; + archive_writer.open(archive_path, FileWriter::OpenMode::CreateForWriting); + + write_archive_metadata(archive_writer, files, timestamp_dict_compressed_size); + size_t metadata_section_size = archive_writer.get_pos() - sizeof(ArchiveHeader); + write_archive_files(archive_writer, files); + m_compressed_size = archive_writer.get_pos(); + write_archive_header(archive_writer, metadata_section_size); + + archive_writer.close(); +} + +void ArchiveWriter::write_archive_metadata( + FileWriter& archive_writer, + std::vector const& files, + size_t timestamp_dict_compressed_size +) { + archive_writer.seek_from_begin(sizeof(ArchiveHeader)); + + ZstdCompressor compressor; + compressor.open(archive_writer, m_compression_level); + compressor.write_numeric_value(3); // Number of packets + + // Write archive info + ArchiveInfoPacket archive_info{.num_segments = 1}; + std::stringstream msgpack_buffer; + msgpack::pack(msgpack_buffer, archive_info); + std::string archive_info_str = msgpack_buffer.str(); + compressor.write_numeric_value(ArchiveMetadataPacketType::ArchiveInfo); + compressor.write_numeric_value(archive_info_str.size()); + compressor.write_string(archive_info_str); + + // Write archive file info + ArchiveFileInfoPacket archive_file_info{.files{files}}; + msgpack_buffer.clear(); + msgpack::pack(msgpack_buffer, archive_file_info); + std::string archive_file_info_str = msgpack_buffer.str(); + compressor.write_numeric_value(ArchiveMetadataPacketType::ArchiveFileInfo); + compressor.write_numeric_value(archive_file_info_str.size()); + compressor.write_string(archive_file_info_str); + + // Write timestamp dictionary + compressor.write_numeric_value(ArchiveMetadataPacketType::TimestampDictionary); + compressor.write_numeric_value(timestamp_dict_compressed_size); + std::string timestamp_dict_path = m_archive_path + constants::cArchiveTimestampDictFile; + FileReader timestamp_dict_reader; + timestamp_dict_reader.open(timestamp_dict_path); + char read_buffer[cReadBlockSize]; + while (true) { + size_t num_bytes_read{0}; + ErrorCode error_code + = timestamp_dict_reader.try_read(read_buffer, cReadBlockSize, num_bytes_read); + if (ErrorCodeSuccess != error_code) { + break; + } + compressor.write(read_buffer, num_bytes_read); + } + timestamp_dict_reader.close(); + std::filesystem::remove(timestamp_dict_path); + compressor.close(); +} + +void ArchiveWriter::write_archive_files( + FileWriter& archive_writer, + std::vector const& files +) { + for (auto const& file : files) { + std::string file_path = m_archive_path + file.n; + FileReader reader; + reader.open(file_path); + char read_buffer[cReadBlockSize]; + while (true) { + size_t num_bytes_read{0}; + ErrorCode const error_code + = reader.try_read(read_buffer, cReadBlockSize, num_bytes_read); + if (ErrorCodeSuccess != error_code) { + break; + } + archive_writer.write(read_buffer, num_bytes_read); + } + reader.close(); + boost::filesystem::remove(file_path); + } +} + +void ArchiveWriter::write_archive_header(FileWriter& archive_writer, size_t metadata_section_size) { + ArchiveHeader header{ + .magic_number{0}, + .version + = (cArchiveMajorVersion << 24) | (cArchiveMinorVersion << 16) | cArchivePatchVersion, + .uncompressed_size = m_uncompressed_size, + .compressed_size = m_compressed_size, + .reserved_padding{0}, + .metadata_section_size = static_cast(metadata_section_size), + .compression_type = static_cast(ArchiveCompressionType::Zstd), + .padding = 0 + }; + std::memcpy(&header.magic_number, "ARCHIVES", sizeof(header.magic_number)); + archive_writer.seek_from_begin(0); + archive_writer.write(reinterpret_cast(&header), sizeof(header)); +} + void ArchiveWriter::append_message( int32_t schema_id, Schema const& schema, @@ -127,8 +259,7 @@ void ArchiveWriter::initialize_schema_writer(SchemaWriter* writer, Schema const& } } -size_t ArchiveWriter::store_tables() { - size_t compressed_size = 0; +std::pair ArchiveWriter::store_tables() { m_tables_file_writer.open( m_archive_path + constants::cArchiveTablesFile, FileWriter::OpenMode::CreateForWriting @@ -153,13 +284,13 @@ size_t ArchiveWriter::store_tables() { } m_table_metadata_compressor.close(); - compressed_size += m_table_metadata_file_writer.get_pos(); - compressed_size += m_tables_file_writer.get_pos(); + auto table_metadata_compressed_size = m_table_metadata_file_writer.get_pos(); + auto table_compressed_size = m_tables_file_writer.get_pos(); m_table_metadata_file_writer.close(); m_tables_file_writer.close(); - return compressed_size; + return {table_metadata_compressed_size, table_compressed_size}; } void ArchiveWriter::update_metadata_db() { diff --git a/components/core/src/clp_s/ArchiveWriter.hpp b/components/core/src/clp_s/ArchiveWriter.hpp index 70eb5dc9b..afb4b3012 100644 --- a/components/core/src/clp_s/ArchiveWriter.hpp +++ b/components/core/src/clp_s/ArchiveWriter.hpp @@ -13,6 +13,7 @@ #include "SchemaMap.hpp" #include "SchemaTree.hpp" #include "SchemaWriter.hpp" +#include "SingleFileArchiveDefs.hpp" #include "TimestampDictionaryWriter.hpp" namespace clp_s { @@ -21,6 +22,7 @@ struct ArchiveWriterOption { std::string archives_dir; int compression_level; bool print_archive_stats; + bool single_file_archive; }; class ArchiveWriter { @@ -109,13 +111,16 @@ class ArchiveWriter { } /** - * Increments the size of the compressed data written to the archive + * Increments the size of the original (uncompressed) logs ingested into the archive. This size + * tracks the raw input size before any encoding or compression. * @param size */ void increment_uncompressed_size(size_t size) { m_uncompressed_size += size; } /** - * @return Size of the uncompressed data written to the archive + * @return The total size of the encoded (uncompressed) data written to the archive. This + * reflects the size of the data after encoding but before compression. + * TODO: Add the size of schema tree, schema map and timestamp dictionary */ size_t get_data_size(); @@ -128,10 +133,48 @@ class ArchiveWriter { void initialize_schema_writer(SchemaWriter* writer, Schema const& schema); /** - * Stores the tables - * @return Size of the compressed data in bytes + * Compresses and stores the tables. + * @return A pair containing: + * - The size of the compressed table metadata in bytes. + * - The size of the compressed tables in bytes. */ - [[nodiscard]] size_t store_tables(); + [[nodiscard]] std::pair store_tables(); + + /** + * Writes the archive to a single file + * @param files + * @param timestamp_dict_compressed_size + */ + void write_single_file_archive( + std::vector const& files, + size_t timestamp_dict_compressed_size + ); + + /** + * Writes the metadata section of the single file archive + * @param archive_writer + * @param files + * @param timestamp_dict_compressed_size + */ + void write_archive_metadata( + FileWriter& archive_writer, + std::vector const& files, + size_t timestamp_dict_compressed_size + ); + + /** + * Writes the file section of the single file archive + * @param archive_writer + * @param files + */ + void write_archive_files(FileWriter& archive_writer, std::vector const& files); + + /** + * Writes the header section of the single file archive + * @param archive_writer + * @param metadata_section_size + */ + void write_archive_header(FileWriter& archive_writer, size_t metadata_section_size); /** * Updates the metadata db with the archive's metadata (id, size, timestamp ranges, etc.) @@ -143,6 +186,8 @@ class ArchiveWriter { */ void print_archive_stats(); + static constexpr size_t cReadBlockSize = 4 * 1024; + size_t m_encoded_message_size{}; size_t m_uncompressed_size{}; size_t m_compressed_size{}; @@ -159,6 +204,7 @@ class ArchiveWriter { std::shared_ptr m_metadata_db; int m_compression_level{}; bool m_print_archive_stats{}; + bool m_single_file_archive{}; SchemaMap m_schema_map; SchemaTree m_schema_tree; diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index 553f17c39..0b77cdebb 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -185,6 +185,10 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { "print-archive-stats", po::bool_switch(&m_print_archive_stats), "Print statistics (json) about the archive after it's compressed." + )( + "single-file-archive", + po::bool_switch(&m_single_file_archive), + "Create a single archive file instead of multiple files." )( "structurize-arrays", po::bool_switch(&m_structurize_arrays), diff --git a/components/core/src/clp_s/CommandLineArguments.hpp b/components/core/src/clp_s/CommandLineArguments.hpp index 030ff8b99..38baa4e39 100644 --- a/components/core/src/clp_s/CommandLineArguments.hpp +++ b/components/core/src/clp_s/CommandLineArguments.hpp @@ -102,6 +102,8 @@ class CommandLineArguments { OutputHandlerType get_output_handler_type() const { return m_output_handler_type; } + bool get_single_file_archive() const { return m_single_file_archive; } + bool get_structurize_arrays() const { return m_structurize_arrays; } bool get_ordered_decompression() const { return m_ordered_decompression; } @@ -172,6 +174,7 @@ class CommandLineArguments { size_t m_target_encoded_size{8ULL * 1024 * 1024 * 1024}; // 8 GiB bool m_print_archive_stats{false}; size_t m_max_document_size{512ULL * 1024 * 1024}; // 512 MB + bool m_single_file_archive{false}; bool m_structurize_arrays{false}; bool m_ordered_decompression{false}; size_t m_ordered_chunk_size{0}; diff --git a/components/core/src/clp_s/JsonParser.cpp b/components/core/src/clp_s/JsonParser.cpp index a68062958..58f9cb425 100644 --- a/components/core/src/clp_s/JsonParser.cpp +++ b/components/core/src/clp_s/JsonParser.cpp @@ -31,6 +31,7 @@ JsonParser::JsonParser(JsonParserOption const& option) m_archive_options.archives_dir = option.archives_dir; m_archive_options.compression_level = option.compression_level; m_archive_options.print_archive_stats = option.print_archive_stats; + m_archive_options.single_file_archive = option.single_file_archive; m_archive_options.id = m_generator(); m_archive_writer = std::make_unique(option.metadata_db); diff --git a/components/core/src/clp_s/JsonParser.hpp b/components/core/src/clp_s/JsonParser.hpp index 84aa27fef..38e7c9d1b 100644 --- a/components/core/src/clp_s/JsonParser.hpp +++ b/components/core/src/clp_s/JsonParser.hpp @@ -34,6 +34,7 @@ struct JsonParserOption { size_t max_document_size; int compression_level; bool print_archive_stats; + bool single_file_archive; bool structurize_arrays; std::shared_ptr metadata_db; }; diff --git a/components/core/src/clp_s/SingleFileArchiveDefs.hpp b/components/core/src/clp_s/SingleFileArchiveDefs.hpp new file mode 100644 index 000000000..c0c4d5a8c --- /dev/null +++ b/components/core/src/clp_s/SingleFileArchiveDefs.hpp @@ -0,0 +1,56 @@ +#ifndef CLP_S_ARCHIVEDEFS_HPP +#define CLP_S_ARCHIVEDEFS_HPP + +#include + +#include "msgpack.hpp" + +namespace clp_s { +// define the version +constexpr uint8_t cArchiveMajorVersion = 0; +constexpr uint8_t cArchiveMinorVersion = 2; +constexpr uint16_t cArchivePatchVersion = 0; + +struct ArchiveHeader { + uint8_t magic_number[8]; + uint32_t version; + uint64_t uncompressed_size; + uint64_t compressed_size; + uint64_t reserved_padding[4]; + uint32_t metadata_section_size; + uint16_t compression_type; + uint16_t padding; +}; + +enum class ArchiveCompressionType : uint16_t { + Zstd = 0, +}; + +enum struct ArchiveMetadataPacketType : uint8_t { + ArchiveInfo = 0, + ArchiveFileInfo = 1, + TimestampDictionary = 2, +}; + +struct ArchiveInfoPacket { + uint64_t num_segments; + // TODO: Add more fields in the future + + MSGPACK_DEFINE(num_segments); +}; + +struct ArchiveFileInfo { + std::string n; // name + uint64_t o; // offset + + MSGPACK_DEFINE(n, o); +}; + +struct ArchiveFileInfoPacket { + std::vector files; + + MSGPACK_DEFINE(files); +}; +} // namespace clp_s + +#endif // CLP_S_ARCHIVEDEFS_HPP diff --git a/components/core/src/clp_s/TimestampDictionaryWriter.cpp b/components/core/src/clp_s/TimestampDictionaryWriter.cpp index 7b02fd3a5..13308eb5a 100644 --- a/components/core/src/clp_s/TimestampDictionaryWriter.cpp +++ b/components/core/src/clp_s/TimestampDictionaryWriter.cpp @@ -57,6 +57,10 @@ size_t TimestampDictionaryWriter::close() { m_dictionary_file_writer.close(); m_is_open = false; + m_next_id = 0; + m_pattern_to_id.clear(); + m_column_id_to_range.clear(); + m_column_key_to_range.clear(); return compressed_size; } diff --git a/components/core/src/clp_s/archive_constants.hpp b/components/core/src/clp_s/archive_constants.hpp index 30e2b78d5..dff598a57 100644 --- a/components/core/src/clp_s/archive_constants.hpp +++ b/components/core/src/clp_s/archive_constants.hpp @@ -2,6 +2,9 @@ #define CLP_S_ARCHIVE_CONSTANTS_HPP namespace clp_s::constants { +// Single file archive +constexpr char cArchiveFile[] = "/archive"; + // Schema files constexpr char cArchiveSchemaMapFile[] = "/schema_ids"; constexpr char cArchiveSchemaTreeFile[] = "/schema_tree"; diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 8e37ca769..ddf9146e5 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -94,6 +94,7 @@ bool compress(CommandLineArguments const& command_line_arguments) { option.compression_level = command_line_arguments.get_compression_level(); option.timestamp_key = command_line_arguments.get_timestamp_key(); option.print_archive_stats = command_line_arguments.print_archive_stats(); + option.single_file_archive = command_line_arguments.get_single_file_archive(); option.structurize_arrays = command_line_arguments.get_structurize_arrays(); auto const& db_config_container = command_line_arguments.get_metadata_db_config(); From 402aef30fbfaf9220f53eff13dafb375b21d2cc8 Mon Sep 17 00:00:00 2001 From: wraymo Date: Mon, 21 Oct 2024 14:19:32 -0400 Subject: [PATCH 02/17] fix lint errors --- components/core/src/clp_s/ArchiveWriter.cpp | 2 +- components/core/src/clp_s/SingleFileArchiveDefs.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp_s/ArchiveWriter.cpp b/components/core/src/clp_s/ArchiveWriter.cpp index c526d5ca5..9b2d2fff3 100644 --- a/components/core/src/clp_s/ArchiveWriter.cpp +++ b/components/core/src/clp_s/ArchiveWriter.cpp @@ -63,7 +63,7 @@ void ArchiveWriter::close() { {constants::cArchiveTablesFile, table_compressed_size} }; uint64_t offset = 0; - for (auto & file : files) { + for (auto& file : files) { uint64_t original_size = file.o; file.o = offset; offset += original_size; diff --git a/components/core/src/clp_s/SingleFileArchiveDefs.hpp b/components/core/src/clp_s/SingleFileArchiveDefs.hpp index c0c4d5a8c..2d8021546 100644 --- a/components/core/src/clp_s/SingleFileArchiveDefs.hpp +++ b/components/core/src/clp_s/SingleFileArchiveDefs.hpp @@ -51,6 +51,6 @@ struct ArchiveFileInfoPacket { MSGPACK_DEFINE(files); }; -} // namespace clp_s +} // namespace clp_s #endif // CLP_S_ARCHIVEDEFS_HPP From 77928446fac703640b2359e90efea05d98f84495 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Tue, 29 Oct 2024 17:53:44 +0000 Subject: [PATCH 03/17] Fix several bugs and spec mismatches --- components/core/src/clp_s/ArchiveWriter.cpp | 36 ++++++++++--------- components/core/src/clp_s/ArchiveWriter.hpp | 14 ++------ .../core/src/clp_s/SingleFileArchiveDefs.hpp | 11 +++--- 3 files changed, 29 insertions(+), 32 deletions(-) diff --git a/components/core/src/clp_s/ArchiveWriter.cpp b/components/core/src/clp_s/ArchiveWriter.cpp index 9b2d2fff3..a31fd3873 100644 --- a/components/core/src/clp_s/ArchiveWriter.cpp +++ b/components/core/src/clp_s/ArchiveWriter.cpp @@ -56,10 +56,10 @@ void ArchiveWriter::close() { std::vector files{ {constants::cArchiveSchemaTreeFile, schema_tree_compressed_size}, {constants::cArchiveSchemaMapFile, schema_map_compressed_size}, + {constants::cArchiveTableMetadataFile, table_metadata_compressed_size}, {constants::cArchiveVarDictFile, var_dict_compressed_size}, {constants::cArchiveLogDictFile, log_dict_compressed_size}, {constants::cArchiveArrayDictFile, array_dict_compressed_size}, - {constants::cArchiveTableMetadataFile, table_metadata_compressed_size}, {constants::cArchiveTablesFile, table_compressed_size} }; uint64_t offset = 0; @@ -68,7 +68,7 @@ void ArchiveWriter::close() { file.o = offset; offset += original_size; } - write_single_file_archive(files, timestamp_dict_compressed_size); + write_single_file_archive(files); } else { m_compressed_size = var_dict_compressed_size + log_dict_compressed_size + array_dict_compressed_size + timestamp_dict_compressed_size @@ -92,15 +92,12 @@ void ArchiveWriter::close() { m_compressed_size = 0UL; } -void ArchiveWriter::write_single_file_archive( - std::vector const& files, - size_t timestamp_dict_compressed_size -) { +void ArchiveWriter::write_single_file_archive(std::vector const& files) { std::string archive_path = m_archive_path + constants::cArchiveFile; FileWriter archive_writer; archive_writer.open(archive_path, FileWriter::OpenMode::CreateForWriting); - write_archive_metadata(archive_writer, files, timestamp_dict_compressed_size); + write_archive_metadata(archive_writer, files); size_t metadata_section_size = archive_writer.get_pos() - sizeof(ArchiveHeader); write_archive_files(archive_writer, files); m_compressed_size = archive_writer.get_pos(); @@ -111,14 +108,13 @@ void ArchiveWriter::write_single_file_archive( void ArchiveWriter::write_archive_metadata( FileWriter& archive_writer, - std::vector const& files, - size_t timestamp_dict_compressed_size + std::vector const& files ) { archive_writer.seek_from_begin(sizeof(ArchiveHeader)); ZstdCompressor compressor; compressor.open(archive_writer, m_compression_level); - compressor.write_numeric_value(3); // Number of packets + compressor.write_numeric_value(static_cast(3U)); // Number of packets // Write archive info ArchiveInfoPacket archive_info{.num_segments = 1}; @@ -126,35 +122,41 @@ void ArchiveWriter::write_archive_metadata( msgpack::pack(msgpack_buffer, archive_info); std::string archive_info_str = msgpack_buffer.str(); compressor.write_numeric_value(ArchiveMetadataPacketType::ArchiveInfo); - compressor.write_numeric_value(archive_info_str.size()); + compressor.write_numeric_value(static_cast(archive_info_str.size())); compressor.write_string(archive_info_str); // Write archive file info ArchiveFileInfoPacket archive_file_info{.files{files}}; - msgpack_buffer.clear(); + msgpack_buffer = std::stringstream{}; msgpack::pack(msgpack_buffer, archive_file_info); std::string archive_file_info_str = msgpack_buffer.str(); compressor.write_numeric_value(ArchiveMetadataPacketType::ArchiveFileInfo); - compressor.write_numeric_value(archive_file_info_str.size()); + compressor.write_numeric_value(static_cast(archive_file_info_str.size())); compressor.write_string(archive_file_info_str); // Write timestamp dictionary compressor.write_numeric_value(ArchiveMetadataPacketType::TimestampDictionary); - compressor.write_numeric_value(timestamp_dict_compressed_size); std::string timestamp_dict_path = m_archive_path + constants::cArchiveTimestampDictFile; FileReader timestamp_dict_reader; + ZstdDecompressor timestamp_dict_decompressor; timestamp_dict_reader.open(timestamp_dict_path); + timestamp_dict_decompressor.open(timestamp_dict_reader, cReadBlockSize); + + std::string decompressed_buffer; char read_buffer[cReadBlockSize]; while (true) { size_t num_bytes_read{0}; ErrorCode error_code - = timestamp_dict_reader.try_read(read_buffer, cReadBlockSize, num_bytes_read); + = timestamp_dict_decompressor.try_read(read_buffer, cReadBlockSize, num_bytes_read); if (ErrorCodeSuccess != error_code) { break; } - compressor.write(read_buffer, num_bytes_read); + decompressed_buffer.append(read_buffer, num_bytes_read); } + timestamp_dict_decompressor.close(); timestamp_dict_reader.close(); + compressor.write_numeric_value(static_cast(decompressed_buffer.size())); + compressor.write(decompressed_buffer.data(), decompressed_buffer.size()); std::filesystem::remove(timestamp_dict_path); compressor.close(); } @@ -194,7 +196,7 @@ void ArchiveWriter::write_archive_header(FileWriter& archive_writer, size_t meta .compression_type = static_cast(ArchiveCompressionType::Zstd), .padding = 0 }; - std::memcpy(&header.magic_number, "ARCHIVES", sizeof(header.magic_number)); + std::memcpy(&header.magic_number, cStructuredSFAMagicNumber, sizeof(header.magic_number)); archive_writer.seek_from_begin(0); archive_writer.write(reinterpret_cast(&header), sizeof(header)); } diff --git a/components/core/src/clp_s/ArchiveWriter.hpp b/components/core/src/clp_s/ArchiveWriter.hpp index afb4b3012..d6acd2d0a 100644 --- a/components/core/src/clp_s/ArchiveWriter.hpp +++ b/components/core/src/clp_s/ArchiveWriter.hpp @@ -143,24 +143,16 @@ class ArchiveWriter { /** * Writes the archive to a single file * @param files - * @param timestamp_dict_compressed_size */ - void write_single_file_archive( - std::vector const& files, - size_t timestamp_dict_compressed_size - ); + void write_single_file_archive(std::vector const& files); /** * Writes the metadata section of the single file archive * @param archive_writer * @param files - * @param timestamp_dict_compressed_size */ - void write_archive_metadata( - FileWriter& archive_writer, - std::vector const& files, - size_t timestamp_dict_compressed_size - ); + void + write_archive_metadata(FileWriter& archive_writer, std::vector const& files); /** * Writes the file section of the single file archive diff --git a/components/core/src/clp_s/SingleFileArchiveDefs.hpp b/components/core/src/clp_s/SingleFileArchiveDefs.hpp index 2d8021546..7eabeb6db 100644 --- a/components/core/src/clp_s/SingleFileArchiveDefs.hpp +++ b/components/core/src/clp_s/SingleFileArchiveDefs.hpp @@ -11,8 +11,11 @@ constexpr uint8_t cArchiveMajorVersion = 0; constexpr uint8_t cArchiveMinorVersion = 2; constexpr uint16_t cArchivePatchVersion = 0; +// define the magic number +constexpr uint8_t cStructuredSFAMagicNumber[] = {0xFD, 0x2F, 0xC5, 0x30}; + struct ArchiveHeader { - uint8_t magic_number[8]; + uint8_t magic_number[4]; uint32_t version; uint64_t uncompressed_size; uint64_t compressed_size; @@ -36,20 +39,20 @@ struct ArchiveInfoPacket { uint64_t num_segments; // TODO: Add more fields in the future - MSGPACK_DEFINE(num_segments); + MSGPACK_DEFINE_MAP(num_segments); }; struct ArchiveFileInfo { std::string n; // name uint64_t o; // offset - MSGPACK_DEFINE(n, o); + MSGPACK_DEFINE_MAP(n, o); }; struct ArchiveFileInfoPacket { std::vector files; - MSGPACK_DEFINE(files); + MSGPACK_DEFINE_MAP(files); }; } // namespace clp_s From a09ab6b47cf614667a963622088f8f6b6bae36f3 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Tue, 5 Nov 2024 22:02:08 +0000 Subject: [PATCH 04/17] Clean up writing code for timestamp dictionary --- components/core/src/clp_s/ArchiveWriter.cpp | 43 +++++++--------- components/core/src/clp_s/ArchiveWriter.hpp | 9 ++++ .../src/clp_s/TimestampDictionaryWriter.cpp | 49 +++---------------- .../src/clp_s/TimestampDictionaryWriter.hpp | 32 ++---------- 4 files changed, 39 insertions(+), 94 deletions(-) diff --git a/components/core/src/clp_s/ArchiveWriter.cpp b/components/core/src/clp_s/ArchiveWriter.cpp index ba286e188..ef1fbe096 100644 --- a/components/core/src/clp_s/ArchiveWriter.cpp +++ b/components/core/src/clp_s/ArchiveWriter.cpp @@ -41,16 +41,13 @@ void ArchiveWriter::open(ArchiveWriterOption const& option) { m_array_dict = std::make_shared(); m_array_dict->open(array_dict_path, m_compression_level, UINT64_MAX); - std::string timestamp_dict_path = m_archive_path + constants::cArchiveTimestampDictFile; m_timestamp_dict = std::make_shared(); - m_timestamp_dict->open(timestamp_dict_path, m_compression_level); } void ArchiveWriter::close() { auto var_dict_compressed_size = m_var_dict->close(); auto log_dict_compressed_size = m_log_dict->close(); auto array_dict_compressed_size = m_array_dict->close(); - auto timestamp_dict_compressed_size = m_timestamp_dict->close(); auto schema_tree_compressed_size = m_schema_tree.store(m_archive_path, m_compression_level); auto schema_map_compressed_size = m_schema_map.store(m_archive_path, m_compression_level); auto [table_metadata_compressed_size, table_compressed_size] = store_tables(); @@ -73,6 +70,9 @@ void ArchiveWriter::close() { } write_single_file_archive(files); } else { + // Timestamp dictionary written separately here until we transition to moving it inside of + // the metadata region of multi-file archives. + auto timestamp_dict_compressed_size = write_timestamp_dict(); m_compressed_size = var_dict_compressed_size + log_dict_compressed_size + array_dict_compressed_size + timestamp_dict_compressed_size + schema_tree_compressed_size + schema_map_compressed_size @@ -95,6 +95,19 @@ void ArchiveWriter::close() { m_compressed_size = 0UL; } +size_t ArchiveWriter::write_timestamp_dict() { + std::string timestamp_dict_path = m_archive_path + constants::cArchiveTimestampDictFile; + FileWriter timestamp_dict_file_writer; + ZstdCompressor timestamp_dict_compressor; + timestamp_dict_file_writer.open(timestamp_dict_path, FileWriter::OpenMode::CreateForWriting); + timestamp_dict_compressor.open(timestamp_dict_file_writer, m_compression_level); + m_timestamp_dict->write(timestamp_dict_compressor); + timestamp_dict_compressor.close(); + auto compressed_size = timestamp_dict_file_writer.get_pos(); + timestamp_dict_file_writer.close(); + return compressed_size; +} + void ArchiveWriter::write_single_file_archive(std::vector const& files) { std::string archive_path = m_archive_path + constants::cArchiveFile; FileWriter archive_writer; @@ -139,28 +152,8 @@ void ArchiveWriter::write_archive_metadata( // Write timestamp dictionary compressor.write_numeric_value(ArchiveMetadataPacketType::TimestampDictionary); - std::string timestamp_dict_path = m_archive_path + constants::cArchiveTimestampDictFile; - FileReader timestamp_dict_reader; - ZstdDecompressor timestamp_dict_decompressor; - timestamp_dict_reader.open(timestamp_dict_path); - timestamp_dict_decompressor.open(timestamp_dict_reader, cReadBlockSize); - - std::string decompressed_buffer; - char read_buffer[cReadBlockSize]; - while (true) { - size_t num_bytes_read{0}; - ErrorCode error_code - = timestamp_dict_decompressor.try_read(read_buffer, cReadBlockSize, num_bytes_read); - if (ErrorCodeSuccess != error_code) { - break; - } - decompressed_buffer.append(read_buffer, num_bytes_read); - } - timestamp_dict_decompressor.close(); - timestamp_dict_reader.close(); - compressor.write_numeric_value(static_cast(decompressed_buffer.size())); - compressor.write(decompressed_buffer.data(), decompressed_buffer.size()); - std::filesystem::remove(timestamp_dict_path); + m_timestamp_dict->write(compressor); + compressor.close(); } diff --git a/components/core/src/clp_s/ArchiveWriter.hpp b/components/core/src/clp_s/ArchiveWriter.hpp index 2a6b779aa..0cc5153d4 100644 --- a/components/core/src/clp_s/ArchiveWriter.hpp +++ b/components/core/src/clp_s/ArchiveWriter.hpp @@ -206,6 +206,15 @@ class ArchiveWriter { */ void print_archive_stats(); + /** + * Write the timestamp dictionary as a dedicated file for multi-file archives. + * + * Note: the timestamp dictionary will be moved into the metadata region of multi-file archives + * in a follow-up PR. + * @return the compressed size of the Timestamp Dictionary in bytes + */ + size_t write_timestamp_dict(); + static constexpr size_t cReadBlockSize = 4 * 1024; size_t m_encoded_message_size{}; diff --git a/components/core/src/clp_s/TimestampDictionaryWriter.cpp b/components/core/src/clp_s/TimestampDictionaryWriter.cpp index 13308eb5a..ee48b13e3 100644 --- a/components/core/src/clp_s/TimestampDictionaryWriter.cpp +++ b/components/core/src/clp_s/TimestampDictionaryWriter.cpp @@ -14,54 +14,19 @@ void TimestampDictionaryWriter::write_timestamp_entries( } } -void TimestampDictionaryWriter::write_and_flush_to_disk() { - write_timestamp_entries(m_column_key_to_range, m_dictionary_compressor); +void TimestampDictionaryWriter::write(ZstdCompressor& compressor) { + merge_range(); + write_timestamp_entries(m_column_key_to_range, compressor); - m_dictionary_compressor.write_numeric_value(m_pattern_to_id.size()); + compressor.write_numeric_value(m_pattern_to_id.size()); for (auto& it : m_pattern_to_id) { // write pattern ID - m_dictionary_compressor.write_numeric_value(it.second); + compressor.write_numeric_value(it.second); std::string const& pattern = it.first->get_format(); - m_dictionary_compressor.write_numeric_value(pattern.length()); - m_dictionary_compressor.write_string(pattern); - } - - m_dictionary_compressor.flush(); - m_dictionary_file_writer.flush(); -} - -void TimestampDictionaryWriter::open(std::string const& dictionary_path, int compression_level) { - if (m_is_open) { - throw OperationFailed(ErrorCodeNotReady, __FILENAME__, __LINE__); - } - - m_dictionary_file_writer.open(dictionary_path, FileWriter::OpenMode::CreateForWriting); - m_dictionary_compressor.open(m_dictionary_file_writer, compression_level); - - m_next_id = 0; - m_is_open = true; -} - -size_t TimestampDictionaryWriter::close() { - if (false == m_is_open) { - throw OperationFailed(ErrorCodeNotInit, __FILENAME__, __LINE__); + compressor.write_numeric_value(pattern.length()); + compressor.write_string(pattern); } - - // merge before writing overall archive because this - // happens before the last sub-archive is written - merge_range(); - write_and_flush_to_disk(); - m_dictionary_compressor.close(); - size_t compressed_size = m_dictionary_file_writer.get_pos(); - m_dictionary_file_writer.close(); - - m_is_open = false; - m_next_id = 0; - m_pattern_to_id.clear(); - m_column_id_to_range.clear(); - m_column_key_to_range.clear(); - return compressed_size; } uint64_t TimestampDictionaryWriter::get_pattern_id(TimestampPattern const* pattern) { diff --git a/components/core/src/clp_s/TimestampDictionaryWriter.hpp b/components/core/src/clp_s/TimestampDictionaryWriter.hpp index 81266b187..291d521dc 100644 --- a/components/core/src/clp_s/TimestampDictionaryWriter.hpp +++ b/components/core/src/clp_s/TimestampDictionaryWriter.hpp @@ -5,7 +5,6 @@ #include #include -#include "FileWriter.hpp" #include "SchemaTree.hpp" #include "TimestampEntry.hpp" #include "TimestampPattern.hpp" @@ -23,25 +22,12 @@ class TimestampDictionaryWriter { }; // Constructors - TimestampDictionaryWriter() : m_is_open(false) {} + TimestampDictionaryWriter() {} /** - * Opens the timestamp dictionary for writing - * @param dictionary_path - * @param compression_level + * Writes the timestamp dictionary to a compression stream. */ - void open(std::string const& dictionary_path, int compression_level); - - /** - * Closes the timestamp dictionary - * @return the compressed size of the global timestamp dictionary in bytes - */ - [[nodiscard]] size_t close(); - - /** - * Writes the timestamp dictionary to disk - */ - void write_and_flush_to_disk(); + void write(ZstdCompressor& compressor); /** * Gets the pattern id for a given pattern @@ -93,12 +79,12 @@ class TimestampDictionaryWriter { private: /** - * Merges timestamp ranges with the same key name + * Merges timestamp ranges with the same key name but different node ids. */ void merge_range(); /** - * Writes timestamp entries to the disk + * Writes timestamp entries to the a compression stream. * @param ranges * @param compressor */ @@ -110,14 +96,6 @@ class TimestampDictionaryWriter { using pattern_to_id_t = std::unordered_map; // Variables - bool m_is_open; - - // Variables related to on-disk storage - FileWriter m_dictionary_file_writer; - ZstdCompressor m_dictionary_compressor; - FileWriter m_dictionary_file_writer_local; - ZstdCompressor m_dictionary_compressor_local; - pattern_to_id_t m_pattern_to_id; uint64_t m_next_id{}; From 53434a7744f13745043185028398171a078d1ed6 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Tue, 5 Nov 2024 22:15:44 +0000 Subject: [PATCH 05/17] Improve error handling --- components/core/src/clp_s/ArchiveWriter.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/core/src/clp_s/ArchiveWriter.cpp b/components/core/src/clp_s/ArchiveWriter.cpp index ef1fbe096..7bac8c700 100644 --- a/components/core/src/clp_s/ArchiveWriter.cpp +++ b/components/core/src/clp_s/ArchiveWriter.cpp @@ -170,8 +170,10 @@ void ArchiveWriter::write_archive_files( size_t num_bytes_read{0}; ErrorCode const error_code = reader.try_read(read_buffer, cReadBlockSize, num_bytes_read); - if (ErrorCodeSuccess != error_code) { + if (ErrorCodeEndOfFile == error_code) { break; + } else if (ErrorCodeSuccess != error_code) { + throw OperationFailed(error_code, __FILENAME__, __LINE__); } archive_writer.write(read_buffer, num_bytes_read); } From 3d51d22d56fcab8e1c5ca8ab424156ee0a1380f8 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Wed, 6 Nov 2024 17:00:25 +0000 Subject: [PATCH 06/17] Add clear method to TimestampDictionaryWriter --- components/core/src/clp_s/ArchiveWriter.cpp | 11 +++++------ components/core/src/clp_s/ArchiveWriter.hpp | 8 ++++---- .../core/src/clp_s/TimestampDictionaryWriter.cpp | 7 +++++++ .../core/src/clp_s/TimestampDictionaryWriter.hpp | 6 ++++++ 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/components/core/src/clp_s/ArchiveWriter.cpp b/components/core/src/clp_s/ArchiveWriter.cpp index 7bac8c700..f45cae5af 100644 --- a/components/core/src/clp_s/ArchiveWriter.cpp +++ b/components/core/src/clp_s/ArchiveWriter.cpp @@ -40,8 +40,6 @@ void ArchiveWriter::open(ArchiveWriterOption const& option) { std::string array_dict_path = m_archive_path + constants::cArchiveArrayDictFile; m_array_dict = std::make_shared(); m_array_dict->open(array_dict_path, m_compression_level, UINT64_MAX); - - m_timestamp_dict = std::make_shared(); } void ArchiveWriter::close() { @@ -90,6 +88,7 @@ void ArchiveWriter::close() { m_id_to_schema_writer.clear(); m_schema_tree.clear(); m_schema_map.clear(); + m_timestamp_dict.clear(); m_encoded_message_size = 0UL; m_uncompressed_size = 0UL; m_compressed_size = 0UL; @@ -101,7 +100,7 @@ size_t ArchiveWriter::write_timestamp_dict() { ZstdCompressor timestamp_dict_compressor; timestamp_dict_file_writer.open(timestamp_dict_path, FileWriter::OpenMode::CreateForWriting); timestamp_dict_compressor.open(timestamp_dict_file_writer, m_compression_level); - m_timestamp_dict->write(timestamp_dict_compressor); + m_timestamp_dict.write(timestamp_dict_compressor); timestamp_dict_compressor.close(); auto compressed_size = timestamp_dict_file_writer.get_pos(); timestamp_dict_file_writer.close(); @@ -152,7 +151,7 @@ void ArchiveWriter::write_archive_metadata( // Write timestamp dictionary compressor.write_numeric_value(ArchiveMetadataPacketType::TimestampDictionary); - m_timestamp_dict->write(compressor); + m_timestamp_dict.write(compressor); compressor.close(); } @@ -388,8 +387,8 @@ void ArchiveWriter::update_metadata_db() { metadata.increment_static_compressed_size(m_compressed_size); metadata.increment_static_uncompressed_size(m_uncompressed_size); metadata.expand_time_range( - m_timestamp_dict->get_begin_timestamp(), - m_timestamp_dict->get_end_timestamp() + m_timestamp_dict.get_begin_timestamp(), + m_timestamp_dict.get_end_timestamp() ); m_metadata_db->add_archive(m_id, metadata); diff --git a/components/core/src/clp_s/ArchiveWriter.hpp b/components/core/src/clp_s/ArchiveWriter.hpp index 0cc5153d4..97a266153 100644 --- a/components/core/src/clp_s/ArchiveWriter.hpp +++ b/components/core/src/clp_s/ArchiveWriter.hpp @@ -121,7 +121,7 @@ class ArchiveWriter { std::string const& timestamp, uint64_t& pattern_id ) { - return m_timestamp_dict->ingest_entry(key, node_id, timestamp, pattern_id); + return m_timestamp_dict.ingest_entry(key, node_id, timestamp, pattern_id); } /** @@ -131,11 +131,11 @@ class ArchiveWriter { * @param timestamp */ void ingest_timestamp_entry(std::string const& key, int32_t node_id, double timestamp) { - m_timestamp_dict->ingest_entry(key, node_id, timestamp); + m_timestamp_dict.ingest_entry(key, node_id, timestamp); } void ingest_timestamp_entry(std::string const& key, int32_t node_id, int64_t timestamp) { - m_timestamp_dict->ingest_entry(key, node_id, timestamp); + m_timestamp_dict.ingest_entry(key, node_id, timestamp); } /** @@ -229,7 +229,7 @@ class ArchiveWriter { std::shared_ptr m_var_dict; std::shared_ptr m_log_dict; std::shared_ptr m_array_dict; // log type dictionary for arrays - std::shared_ptr m_timestamp_dict; + TimestampDictionaryWriter m_timestamp_dict; std::shared_ptr m_metadata_db; int m_compression_level{}; bool m_print_archive_stats{}; diff --git a/components/core/src/clp_s/TimestampDictionaryWriter.cpp b/components/core/src/clp_s/TimestampDictionaryWriter.cpp index ee48b13e3..c79953150 100644 --- a/components/core/src/clp_s/TimestampDictionaryWriter.cpp +++ b/components/core/src/clp_s/TimestampDictionaryWriter.cpp @@ -149,4 +149,11 @@ epochtime_t TimestampDictionaryWriter::get_end_timestamp() const { return it->second.get_end_timestamp(); } + +void TimestampDictionaryWriter::clear() { + m_next_id = 0; + m_pattern_to_id.clear(); + m_column_key_to_range.clear(); + m_column_id_to_range.clear(); +} } // namespace clp_s diff --git a/components/core/src/clp_s/TimestampDictionaryWriter.hpp b/components/core/src/clp_s/TimestampDictionaryWriter.hpp index 291d521dc..38faac701 100644 --- a/components/core/src/clp_s/TimestampDictionaryWriter.hpp +++ b/components/core/src/clp_s/TimestampDictionaryWriter.hpp @@ -1,6 +1,7 @@ #ifndef CLP_S_TIMESTAMPDICTIONARYWRITER_HPP #define CLP_S_TIMESTAMPDICTIONARYWRITER_HPP +#include #include #include #include @@ -77,6 +78,11 @@ class TimestampDictionaryWriter { */ epochtime_t get_end_timestamp() const; + /** + * Clears and resets all internal state. + */ + void clear(); + private: /** * Merges timestamp ranges with the same key name but different node ids. From eaa09825f9c2c94f94e241c58d6cfed8572b8568 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Thu, 7 Nov 2024 16:36:24 +0000 Subject: [PATCH 07/17] Fix bug where size of timestamp dictionary section was missing --- components/core/src/clp_s/ArchiveWriter.cpp | 1 + components/core/src/clp_s/DictionaryReader.hpp | 7 +++++-- .../core/src/clp_s/TimestampDictionaryWriter.cpp | 14 ++++++++++++++ .../core/src/clp_s/TimestampDictionaryWriter.hpp | 5 +++++ components/core/src/clp_s/TimestampEntry.hpp | 6 ++++++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp_s/ArchiveWriter.cpp b/components/core/src/clp_s/ArchiveWriter.cpp index f45cae5af..b65632fb1 100644 --- a/components/core/src/clp_s/ArchiveWriter.cpp +++ b/components/core/src/clp_s/ArchiveWriter.cpp @@ -151,6 +151,7 @@ void ArchiveWriter::write_archive_metadata( // Write timestamp dictionary compressor.write_numeric_value(ArchiveMetadataPacketType::TimestampDictionary); + compressor.write_numeric_value(static_cast(m_timestamp_dict.size_in_bytes())); m_timestamp_dict.write(compressor); compressor.close(); diff --git a/components/core/src/clp_s/DictionaryReader.hpp b/components/core/src/clp_s/DictionaryReader.hpp index 175214d88..764a25306 100644 --- a/components/core/src/clp_s/DictionaryReader.hpp +++ b/components/core/src/clp_s/DictionaryReader.hpp @@ -87,9 +87,12 @@ class DictionaryReader { std::vector m_entries; }; -class VariableDictionaryReader : public DictionaryReader {}; +using VariableDictionaryReader = DictionaryReader; +using LogTypeDictionaryReader = DictionaryReader; -class LogTypeDictionaryReader : public DictionaryReader {}; +// class VariableDictionaryReader : public DictionaryReader {}; + +// class LogTypeDictionaryReader : public DictionaryReader {}; template void DictionaryReader::open(std::string const& dictionary_path) { diff --git a/components/core/src/clp_s/TimestampDictionaryWriter.cpp b/components/core/src/clp_s/TimestampDictionaryWriter.cpp index c79953150..7e8a36768 100644 --- a/components/core/src/clp_s/TimestampDictionaryWriter.cpp +++ b/components/core/src/clp_s/TimestampDictionaryWriter.cpp @@ -156,4 +156,18 @@ void TimestampDictionaryWriter::clear() { m_column_key_to_range.clear(); m_column_id_to_range.clear(); } + +size_t TimestampDictionaryWriter::size_in_bytes() { + merge_range(); + size_t size{2 * sizeof(uint64_t)}; + for (auto const& range : m_column_key_to_range) { + size += range.second.size_in_bytes(); + } + + for (auto& pattern : m_pattern_to_id) { + size += 2 * sizeof(uint64_t); + size += pattern.first->get_format().size(); + } + return size; +} } // namespace clp_s diff --git a/components/core/src/clp_s/TimestampDictionaryWriter.hpp b/components/core/src/clp_s/TimestampDictionaryWriter.hpp index 38faac701..a28bfad7c 100644 --- a/components/core/src/clp_s/TimestampDictionaryWriter.hpp +++ b/components/core/src/clp_s/TimestampDictionaryWriter.hpp @@ -83,6 +83,11 @@ class TimestampDictionaryWriter { */ void clear(); + /** + * Merge ranges by key name then return the size of data to be compressed in bytes + */ + size_t size_in_bytes(); + private: /** * Merges timestamp ranges with the same key name but different node ids. diff --git a/components/core/src/clp_s/TimestampEntry.hpp b/components/core/src/clp_s/TimestampEntry.hpp index ad40b4b89..0b5bac78c 100644 --- a/components/core/src/clp_s/TimestampEntry.hpp +++ b/components/core/src/clp_s/TimestampEntry.hpp @@ -119,6 +119,12 @@ class TimestampEntry { */ epochtime_t get_end_timestamp() const; + size_t size_in_bytes() const { + return sizeof(uint64_t) + m_key_name.size() + sizeof(uint64_t) + + m_column_ids.size() * sizeof(int32_t) + sizeof(TimestampEncoding) + + 2 * sizeof(epochtime_t); + } + private: TimestampEncoding m_encoding; double m_epoch_start_double, m_epoch_end_double; From 5f774c35d9cf390e45bd73af460377790fa104ac Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Thu, 7 Nov 2024 16:43:31 +0000 Subject: [PATCH 08/17] Revert change accidentally pulled into this PR --- components/core/src/clp_s/DictionaryReader.hpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/components/core/src/clp_s/DictionaryReader.hpp b/components/core/src/clp_s/DictionaryReader.hpp index 764a25306..175214d88 100644 --- a/components/core/src/clp_s/DictionaryReader.hpp +++ b/components/core/src/clp_s/DictionaryReader.hpp @@ -87,12 +87,9 @@ class DictionaryReader { std::vector m_entries; }; -using VariableDictionaryReader = DictionaryReader; -using LogTypeDictionaryReader = DictionaryReader; +class VariableDictionaryReader : public DictionaryReader {}; -// class VariableDictionaryReader : public DictionaryReader {}; - -// class LogTypeDictionaryReader : public DictionaryReader {}; +class LogTypeDictionaryReader : public DictionaryReader {}; template void DictionaryReader::open(std::string const& dictionary_path) { From bec587b03de07d3d6404f25db97f45823a57a578 Mon Sep 17 00:00:00 2001 From: Devin Gibson Date: Sun, 24 Nov 2024 17:16:33 -0500 Subject: [PATCH 09/17] Update components/core/src/clp_s/TimestampDictionaryWriter.hpp Co-authored-by: wraymo <37269683+wraymo@users.noreply.github.com> --- components/core/src/clp_s/TimestampDictionaryWriter.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp_s/TimestampDictionaryWriter.hpp b/components/core/src/clp_s/TimestampDictionaryWriter.hpp index a28bfad7c..6012c4c77 100644 --- a/components/core/src/clp_s/TimestampDictionaryWriter.hpp +++ b/components/core/src/clp_s/TimestampDictionaryWriter.hpp @@ -95,7 +95,7 @@ class TimestampDictionaryWriter { void merge_range(); /** - * Writes timestamp entries to the a compression stream. + * Writes timestamp entries to a compression stream. * @param ranges * @param compressor */ From 816526ec9e8f7efacc0ade567d529c07dfda51f1 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Sun, 24 Nov 2024 22:20:56 +0000 Subject: [PATCH 10/17] Address review comments --- components/core/src/clp_s/ArchiveWriter.cpp | 2 +- components/core/src/clp_s/TimestampDictionaryWriter.hpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/components/core/src/clp_s/ArchiveWriter.cpp b/components/core/src/clp_s/ArchiveWriter.cpp index b65632fb1..11a12f057 100644 --- a/components/core/src/clp_s/ArchiveWriter.cpp +++ b/components/core/src/clp_s/ArchiveWriter.cpp @@ -161,9 +161,9 @@ void ArchiveWriter::write_archive_files( FileWriter& archive_writer, std::vector const& files ) { + FileReader reader; for (auto const& file : files) { std::string file_path = m_archive_path + file.n; - FileReader reader; reader.open(file_path); char read_buffer[cReadBlockSize]; while (true) { diff --git a/components/core/src/clp_s/TimestampDictionaryWriter.hpp b/components/core/src/clp_s/TimestampDictionaryWriter.hpp index 6012c4c77..daf16df48 100644 --- a/components/core/src/clp_s/TimestampDictionaryWriter.hpp +++ b/components/core/src/clp_s/TimestampDictionaryWriter.hpp @@ -27,6 +27,7 @@ class TimestampDictionaryWriter { /** * Writes the timestamp dictionary to a compression stream. + * @param compressor */ void write(ZstdCompressor& compressor); From d0167d48fe5f26ab5863875d77efb0fe9bd9c670 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 25 Nov 2024 17:26:50 +0000 Subject: [PATCH 11/17] Add back option deleted during merge --- components/core/src/clp_s/JsonParser.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/core/src/clp_s/JsonParser.hpp b/components/core/src/clp_s/JsonParser.hpp index d7cc5a2fe..bfd423c22 100644 --- a/components/core/src/clp_s/JsonParser.hpp +++ b/components/core/src/clp_s/JsonParser.hpp @@ -38,6 +38,7 @@ struct JsonParserOption { bool print_archive_stats{}; bool structurize_arrays{}; bool record_log_order{true}; + bool single_file_archive{false}; std::shared_ptr metadata_db; }; From ef917472ae8b45b7cede8c69720eac9bbdcc0ee0 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 25 Nov 2024 17:48:21 +0000 Subject: [PATCH 12/17] Write single file archive directly in archives_dir instead of subdirectory --- components/core/src/clp_s/ArchiveWriter.cpp | 27 +++++++++++++------ components/core/src/clp_s/ArchiveWriter.hpp | 1 + .../core/src/clp_s/archive_constants.hpp | 2 +- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/components/core/src/clp_s/ArchiveWriter.cpp b/components/core/src/clp_s/ArchiveWriter.cpp index 78ac6fff8..26883685c 100644 --- a/components/core/src/clp_s/ArchiveWriter.cpp +++ b/components/core/src/clp_s/ArchiveWriter.cpp @@ -1,6 +1,7 @@ #include "ArchiveWriter.hpp" #include +#include #include @@ -15,17 +16,21 @@ void ArchiveWriter::open(ArchiveWriterOption const& option) { m_print_archive_stats = option.print_archive_stats; m_single_file_archive = option.single_file_archive; m_min_table_size = option.min_table_size; - auto archive_path = boost::filesystem::path(option.archives_dir) / m_id; + m_archives_dir = option.archives_dir; + std::string working_dir_name = m_id; + if (option.single_file_archive) { + working_dir_name += constants::cTmpPostfix; + } + auto archive_path = std::filesystem::path(option.archives_dir) / working_dir_name; - boost::system::error_code boost_error_code; - bool path_exists = boost::filesystem::exists(archive_path, boost_error_code); - if (path_exists) { + std::error_code ec; + if (std::filesystem::exists(archive_path, ec)) { SPDLOG_ERROR("Archive path already exists: {}", archive_path.c_str()); throw OperationFailed(ErrorCodeUnsupported, __FILENAME__, __LINE__); } m_archive_path = archive_path.string(); - if (false == boost::filesystem::create_directory(m_archive_path)) { + if (false == std::filesystem::create_directory(m_archive_path, ec)) { throw OperationFailed(ErrorCodeErrno, __FILENAME__, __LINE__); } @@ -109,9 +114,9 @@ size_t ArchiveWriter::write_timestamp_dict() { } void ArchiveWriter::write_single_file_archive(std::vector const& files) { - std::string archive_path = m_archive_path + constants::cArchiveFile; + std::string single_file_archive_path = (std::filesystem::path(m_archives_dir) / m_id).string(); FileWriter archive_writer; - archive_writer.open(archive_path, FileWriter::OpenMode::CreateForWriting); + archive_writer.open(single_file_archive_path, FileWriter::OpenMode::CreateForWriting); write_archive_metadata(archive_writer, files); size_t metadata_section_size = archive_writer.get_pos() - sizeof(ArchiveHeader); @@ -120,6 +125,10 @@ void ArchiveWriter::write_single_file_archive(std::vector const write_archive_header(archive_writer, metadata_section_size); archive_writer.close(); + std::error_code ec; + if (false == std::filesystem::remove(m_archive_path, ec)) { + throw OperationFailed(ErrorCodeFileExists, __FILENAME__, __LINE__); + } } void ArchiveWriter::write_archive_metadata( @@ -179,7 +188,9 @@ void ArchiveWriter::write_archive_files( archive_writer.write(read_buffer, num_bytes_read); } reader.close(); - boost::filesystem::remove(file_path); + if (false == std::filesystem::remove(file_path)) { + throw OperationFailed(ErrorCodeFileExists, __FILENAME__, __LINE__); + } } } diff --git a/components/core/src/clp_s/ArchiveWriter.hpp b/components/core/src/clp_s/ArchiveWriter.hpp index da67faf4d..3b13f4426 100644 --- a/components/core/src/clp_s/ArchiveWriter.hpp +++ b/components/core/src/clp_s/ArchiveWriter.hpp @@ -230,6 +230,7 @@ class ArchiveWriter { std::string m_id; + std::string m_archives_dir; std::string m_archive_path; std::string m_encoded_messages_dir; diff --git a/components/core/src/clp_s/archive_constants.hpp b/components/core/src/clp_s/archive_constants.hpp index b38f40713..b76af2944 100644 --- a/components/core/src/clp_s/archive_constants.hpp +++ b/components/core/src/clp_s/archive_constants.hpp @@ -5,7 +5,7 @@ namespace clp_s::constants { // Single file archive -constexpr char cArchiveFile[] = "/archive"; +constexpr char cTmpPostfix[] = ".tmp"; // Schema files constexpr char cArchiveSchemaMapFile[] = "/schema_ids"; From 62c43bdd90270c2aa568505520849d1a9894c08c Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 25 Nov 2024 18:15:44 +0000 Subject: [PATCH 13/17] Write timestamp dictionary to a buffered stream instead of directly to a ZstdCompressor --- components/core/src/clp_s/ArchiveWriter.cpp | 13 ++++-- .../src/clp_s/TimestampDictionaryWriter.cpp | 41 ++++++++----------- .../src/clp_s/TimestampDictionaryWriter.hpp | 17 +++----- components/core/src/clp_s/TimestampEntry.cpp | 28 ++++++++----- components/core/src/clp_s/TimestampEntry.hpp | 12 ++---- 5 files changed, 55 insertions(+), 56 deletions(-) diff --git a/components/core/src/clp_s/ArchiveWriter.cpp b/components/core/src/clp_s/ArchiveWriter.cpp index 26883685c..d627479de 100644 --- a/components/core/src/clp_s/ArchiveWriter.cpp +++ b/components/core/src/clp_s/ArchiveWriter.cpp @@ -2,6 +2,7 @@ #include #include +#include #include @@ -106,7 +107,10 @@ size_t ArchiveWriter::write_timestamp_dict() { ZstdCompressor timestamp_dict_compressor; timestamp_dict_file_writer.open(timestamp_dict_path, FileWriter::OpenMode::CreateForWriting); timestamp_dict_compressor.open(timestamp_dict_file_writer, m_compression_level); - m_timestamp_dict.write(timestamp_dict_compressor); + std::stringstream timestamp_dict_stream; + m_timestamp_dict.write(timestamp_dict_stream); + std::string encoded_timestamp_dict = timestamp_dict_stream.str(); + timestamp_dict_compressor.write(encoded_timestamp_dict.data(), encoded_timestamp_dict.size()); timestamp_dict_compressor.close(); auto compressed_size = timestamp_dict_file_writer.get_pos(); timestamp_dict_file_writer.close(); @@ -161,8 +165,11 @@ void ArchiveWriter::write_archive_metadata( // Write timestamp dictionary compressor.write_numeric_value(ArchiveMetadataPacketType::TimestampDictionary); - compressor.write_numeric_value(static_cast(m_timestamp_dict.size_in_bytes())); - m_timestamp_dict.write(compressor); + std::stringstream timestamp_dict_stream; + m_timestamp_dict.write(timestamp_dict_stream); + std::string encoded_timestamp_dict = timestamp_dict_stream.str(); + compressor.write_numeric_value(static_cast(encoded_timestamp_dict.size())); + compressor.write(encoded_timestamp_dict.data(), encoded_timestamp_dict.size()); compressor.close(); } diff --git a/components/core/src/clp_s/TimestampDictionaryWriter.cpp b/components/core/src/clp_s/TimestampDictionaryWriter.cpp index 7e8a36768..f757790ae 100644 --- a/components/core/src/clp_s/TimestampDictionaryWriter.cpp +++ b/components/core/src/clp_s/TimestampDictionaryWriter.cpp @@ -1,31 +1,40 @@ #include "TimestampDictionaryWriter.hpp" +#include + #include "Utils.hpp" +namespace { +template +void write_numeric_value(std::stringstream& stream, T value) { + stream.write(reinterpret_cast(&value), sizeof(value)); +} +} // namespace + namespace clp_s { void TimestampDictionaryWriter::write_timestamp_entries( std::map const& ranges, - ZstdCompressor& compressor + std::stringstream& stream ) { - compressor.write_numeric_value(ranges.size()); + write_numeric_value(stream, ranges.size()); for (auto const& range : ranges) { - range.second.write_to_file(compressor); + range.second.write_to_stream(stream); } } -void TimestampDictionaryWriter::write(ZstdCompressor& compressor) { +void TimestampDictionaryWriter::write(std::stringstream& stream) { merge_range(); - write_timestamp_entries(m_column_key_to_range, compressor); + write_timestamp_entries(m_column_key_to_range, stream); - compressor.write_numeric_value(m_pattern_to_id.size()); + write_numeric_value(stream, m_pattern_to_id.size()); for (auto& it : m_pattern_to_id) { // write pattern ID - compressor.write_numeric_value(it.second); + write_numeric_value(stream, it.second); std::string const& pattern = it.first->get_format(); - compressor.write_numeric_value(pattern.length()); - compressor.write_string(pattern); + write_numeric_value(stream, pattern.length()); + stream.write(pattern.data(), pattern.size()); } } @@ -156,18 +165,4 @@ void TimestampDictionaryWriter::clear() { m_column_key_to_range.clear(); m_column_id_to_range.clear(); } - -size_t TimestampDictionaryWriter::size_in_bytes() { - merge_range(); - size_t size{2 * sizeof(uint64_t)}; - for (auto const& range : m_column_key_to_range) { - size += range.second.size_in_bytes(); - } - - for (auto& pattern : m_pattern_to_id) { - size += 2 * sizeof(uint64_t); - size += pattern.first->get_format().size(); - } - return size; -} } // namespace clp_s diff --git a/components/core/src/clp_s/TimestampDictionaryWriter.hpp b/components/core/src/clp_s/TimestampDictionaryWriter.hpp index daf16df48..29288fd48 100644 --- a/components/core/src/clp_s/TimestampDictionaryWriter.hpp +++ b/components/core/src/clp_s/TimestampDictionaryWriter.hpp @@ -2,6 +2,7 @@ #define CLP_S_TIMESTAMPDICTIONARYWRITER_HPP #include +#include #include #include #include @@ -9,7 +10,6 @@ #include "SchemaTree.hpp" #include "TimestampEntry.hpp" #include "TimestampPattern.hpp" -#include "ZstdCompressor.hpp" namespace clp_s { class TimestampDictionaryWriter { @@ -26,10 +26,10 @@ class TimestampDictionaryWriter { TimestampDictionaryWriter() {} /** - * Writes the timestamp dictionary to a compression stream. - * @param compressor + * Writes the timestamp dictionary to a buffered stream. + * @param stream */ - void write(ZstdCompressor& compressor); + void write(std::stringstream& stream); /** * Gets the pattern id for a given pattern @@ -84,11 +84,6 @@ class TimestampDictionaryWriter { */ void clear(); - /** - * Merge ranges by key name then return the size of data to be compressed in bytes - */ - size_t size_in_bytes(); - private: /** * Merges timestamp ranges with the same key name but different node ids. @@ -96,13 +91,13 @@ class TimestampDictionaryWriter { void merge_range(); /** - * Writes timestamp entries to a compression stream. + * Writes timestamp entries to a buffered stream. * @param ranges * @param compressor */ static void write_timestamp_entries( std::map const& ranges, - ZstdCompressor& compressor + std::stringstream& stream ); using pattern_to_id_t = std::unordered_map; diff --git a/components/core/src/clp_s/TimestampEntry.cpp b/components/core/src/clp_s/TimestampEntry.cpp index 54b27d22e..54677117b 100644 --- a/components/core/src/clp_s/TimestampEntry.cpp +++ b/components/core/src/clp_s/TimestampEntry.cpp @@ -1,6 +1,14 @@ #include "TimestampEntry.hpp" #include +#include + +namespace { +template +void write_numeric_value(std::stringstream& stream, T value) { + stream.write(reinterpret_cast(&value), sizeof(value)); +} +} // namespace namespace clp_s { void TimestampEntry::ingest_timestamp(epochtime_t timestamp) { @@ -54,21 +62,21 @@ void TimestampEntry::merge_range(TimestampEntry const& entry) { } } -void TimestampEntry::write_to_file(ZstdCompressor& compressor) const { - compressor.write_numeric_value(m_key_name.size()); - compressor.write_string(m_key_name); - compressor.write_numeric_value(m_column_ids.size()); +void TimestampEntry::write_to_stream(std::stringstream& stream) const { + write_numeric_value(stream, m_key_name.size()); + stream.write(m_key_name.data(), m_key_name.size()); + write_numeric_value(stream, m_column_ids.size()); for (auto const& id : m_column_ids) { - compressor.write_numeric_value(id); + write_numeric_value(stream, id); } - compressor.write_numeric_value(m_encoding); + write_numeric_value(stream, m_encoding); if (m_encoding == Epoch) { - compressor.write_numeric_value(m_epoch_start); - compressor.write_numeric_value(m_epoch_end); + write_numeric_value(stream, m_epoch_start); + write_numeric_value(stream, m_epoch_end); } else if (m_encoding == DoubleEpoch) { - compressor.write_numeric_value(m_epoch_start_double); - compressor.write_numeric_value(m_epoch_end_double); + write_numeric_value(stream, m_epoch_start_double); + write_numeric_value(stream, m_epoch_end_double); } } diff --git a/components/core/src/clp_s/TimestampEntry.hpp b/components/core/src/clp_s/TimestampEntry.hpp index 0b5bac78c..326ed9d73 100644 --- a/components/core/src/clp_s/TimestampEntry.hpp +++ b/components/core/src/clp_s/TimestampEntry.hpp @@ -1,6 +1,7 @@ #ifndef CLP_S_TIMESTAMPENTRY_HPP #define CLP_S_TIMESTAMPENTRY_HPP +#include #include #include #include @@ -9,7 +10,6 @@ #include "ErrorCode.hpp" #include "search/FilterOperation.hpp" #include "Utils.hpp" -#include "ZstdCompressor.hpp" #include "ZstdDecompressor.hpp" using clp_s::search::FilterOperation; @@ -66,10 +66,10 @@ class TimestampEntry { void merge_range(TimestampEntry const& entry); /** - * Write the timestamp entry to a file + * Write the timestamp entry to a buffered stream. * @param compressor */ - void write_to_file(ZstdCompressor& compressor) const; + void write_to_stream(std::stringstream& stream) const; /** * Try to read the timestamp entry from a file @@ -119,12 +119,6 @@ class TimestampEntry { */ epochtime_t get_end_timestamp() const; - size_t size_in_bytes() const { - return sizeof(uint64_t) + m_key_name.size() + sizeof(uint64_t) - + m_column_ids.size() * sizeof(int32_t) + sizeof(TimestampEncoding) - + 2 * sizeof(epochtime_t); - } - private: TimestampEncoding m_encoding; double m_epoch_start_double, m_epoch_end_double; From 4c73e299a0af3144337a676e6d0e214facc36081 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Tue, 26 Nov 2024 11:37:32 -0500 Subject: [PATCH 14/17] Remove duplicated utility code --- components/core/src/clp_s/TimestampDictionaryWriter.cpp | 7 ------- components/core/src/clp_s/TimestampEntry.cpp | 7 +------ components/core/src/clp_s/Utils.hpp | 5 +++++ 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/components/core/src/clp_s/TimestampDictionaryWriter.cpp b/components/core/src/clp_s/TimestampDictionaryWriter.cpp index f757790ae..39e66a6af 100644 --- a/components/core/src/clp_s/TimestampDictionaryWriter.cpp +++ b/components/core/src/clp_s/TimestampDictionaryWriter.cpp @@ -4,13 +4,6 @@ #include "Utils.hpp" -namespace { -template -void write_numeric_value(std::stringstream& stream, T value) { - stream.write(reinterpret_cast(&value), sizeof(value)); -} -} // namespace - namespace clp_s { void TimestampDictionaryWriter::write_timestamp_entries( std::map const& ranges, diff --git a/components/core/src/clp_s/TimestampEntry.cpp b/components/core/src/clp_s/TimestampEntry.cpp index 54677117b..19d422066 100644 --- a/components/core/src/clp_s/TimestampEntry.cpp +++ b/components/core/src/clp_s/TimestampEntry.cpp @@ -3,12 +3,7 @@ #include #include -namespace { -template -void write_numeric_value(std::stringstream& stream, T value) { - stream.write(reinterpret_cast(&value), sizeof(value)); -} -} // namespace +#include "Utils.hpp" namespace clp_s { void TimestampEntry::ingest_timestamp(epochtime_t timestamp) { diff --git a/components/core/src/clp_s/Utils.hpp b/components/core/src/clp_s/Utils.hpp index d6deb3280..9659f9d80 100644 --- a/components/core/src/clp_s/Utils.hpp +++ b/components/core/src/clp_s/Utils.hpp @@ -254,6 +254,11 @@ inline T2 bit_cast(T1 t1) { return t2; } +template +void write_numeric_value(std::stringstream& stream, T value) { + stream.write(reinterpret_cast(&value), sizeof(value)); +} + /** * A span of memory where the underlying memory may not be aligned correctly for type T. * From 1e2be7ecfd5d46428eb773ff82a85535fd958a27 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Tue, 26 Nov 2024 11:39:22 -0500 Subject: [PATCH 15/17] Add docstring for new utility --- components/core/src/clp_s/Utils.hpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp_s/Utils.hpp b/components/core/src/clp_s/Utils.hpp index 9659f9d80..b3d601b63 100644 --- a/components/core/src/clp_s/Utils.hpp +++ b/components/core/src/clp_s/Utils.hpp @@ -254,8 +254,13 @@ inline T2 bit_cast(T1 t1) { return t2; } -template -void write_numeric_value(std::stringstream& stream, T value) { +/** + * Writes a numeric value to a stringstream. + * @param val + * @tparam ValueType + */ +template +void write_numeric_value(std::stringstream& stream, ValueType value) { stream.write(reinterpret_cast(&value), sizeof(value)); } From 4c7a50ffc69f054f8e4584ecffa0ce34be0ec88a Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Tue, 26 Nov 2024 12:09:02 -0500 Subject: [PATCH 16/17] Fix docstring --- components/core/src/clp_s/Utils.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/core/src/clp_s/Utils.hpp b/components/core/src/clp_s/Utils.hpp index b3d601b63..a6b637106 100644 --- a/components/core/src/clp_s/Utils.hpp +++ b/components/core/src/clp_s/Utils.hpp @@ -256,7 +256,8 @@ inline T2 bit_cast(T1 t1) { /** * Writes a numeric value to a stringstream. - * @param val + * @param stream + * @param value * @tparam ValueType */ template From c6984876cfc63516470404945b35cab7526caf6d Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Wed, 27 Nov 2024 15:51:33 +0000 Subject: [PATCH 17/17] Fix build issues --- components/core/src/clp_s/Utils.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/core/src/clp_s/Utils.hpp b/components/core/src/clp_s/Utils.hpp index a6b637106..553f7e608 100644 --- a/components/core/src/clp_s/Utils.hpp +++ b/components/core/src/clp_s/Utils.hpp @@ -3,6 +3,7 @@ #include #include +#include #include #include