diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index be04efd6f4..b0b71528e8 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.21) # TARGET_RUNTIME_DLLS # Make the `project` command handle the version of the project. cmake_policy(SET CMP0048 NEW) -set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) # We do not need any compilers' extensions, so we disable them. set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/cpp/arcticdb/CMakeLists.txt b/cpp/arcticdb/CMakeLists.txt index 9e1eb0daeb..6149a0709d 100644 --- a/cpp/arcticdb/CMakeLists.txt +++ b/cpp/arcticdb/CMakeLists.txt @@ -347,7 +347,6 @@ set(arcticdb_srcs util/memory_mapped_file.hpp util/name_validation.hpp util/offset_string.hpp - util/offset_string.hpp util/optional_defaults.hpp util/pb_util.hpp util/preconditions.hpp @@ -473,7 +472,6 @@ set(arcticdb_srcs util/memory_mapped_file.hpp util/name_validation.cpp util/offset_string.cpp - util/offset_string.cpp util/sparse_utils.cpp util/string_utils.cpp util/trace.cpp @@ -488,7 +486,7 @@ set(arcticdb_srcs version/symbol_list.cpp version/version_map_batch_methods.cpp storage/s3/ec2_utils.cpp -) + storage/lmdb/lmdb.hpp util/cxx17_concepts.hpp) if(${ARCTICDB_INCLUDE_ROCKSDB}) list (APPEND arcticdb_srcs @@ -680,6 +678,7 @@ target_include_directories(arcticdb_python PRIVATE ${LIBMONGOCXX_STATIC_INCLUDE_DIRS} ${LIBBSONCXX_STATIC_INCLUDE_DIRS} ${BITMAGIC_INCLUDE_DIRS} + ${LMDB_LIBRARIES} ) diff --git a/cpp/arcticdb/column_store/column.hpp b/cpp/arcticdb/column_store/column.hpp index 7d7a11cdbe..2eab601111 100644 --- a/cpp/arcticdb/column_store/column.hpp +++ b/cpp/arcticdb/column_store/column.hpp @@ -28,10 +28,11 @@ #include #include -#include #include #include +#include + namespace py = pybind11; namespace arcticdb { @@ -677,7 +678,7 @@ class Column { template < typename input_tdt, typename functor> - requires std::is_invocable_r_v + // requires std::is_invocable_r_v //TODO reinstate with C++20 static void for_each(const Column& input_column, functor&& f) { auto input_data = input_column.data(); std::for_each(input_data.cbegin(), input_data.cend(), std::forward(f)); @@ -686,7 +687,7 @@ class Column { template < typename input_tdt, typename functor> - requires std::is_invocable_r_v> + //requires std::is_invocable_r_v> static void for_each_enumerated(const Column& input_column, functor&& f) { auto input_data = input_column.data(); if (input_column.is_sparse()) { @@ -699,10 +700,13 @@ class Column { } template < - typename input_tdt, - typename output_tdt, - typename functor> - requires std::is_invocable_r_v + typename input_tdt, + typename output_tdt, + typename functor, + typename = std::enable_if< + std::is_invocable_r_v< + typename output_tdt::DataTypeTag::raw_type, functor, + typename input_tdt::DataTypeTag::raw_type>>> static void transform(const Column& input_column, Column& output_column, functor&& f) { auto input_data = input_column.data(); initialise_output_column(input_column, output_column); @@ -719,12 +723,12 @@ class Column { typename left_input_tdt, typename right_input_tdt, typename output_tdt, - typename functor> - requires std::is_invocable_r_v< - typename output_tdt::DataTypeTag::raw_type, - functor, - typename left_input_tdt::DataTypeTag::raw_type, - typename right_input_tdt::DataTypeTag::raw_type> + typename functor, + typename = std::enable_if>> static void transform(const Column& left_input_column, const Column& right_input_column, Column& output_column, @@ -783,8 +787,8 @@ class Column { template < typename input_tdt, - std::predicate functor> - static void transform(const Column& input_column, + typename functor> + static void transform_to_bitset(const Column& input_column, util::BitSet& output_bitset, bool sparse_missing_value_output, functor&& f) { @@ -806,7 +810,7 @@ class Column { template < typename left_input_tdt, typename right_input_tdt, - std::relation functor> + typename functor> static void transform(const Column& left_input_column, const Column& right_input_column, util::BitSet& output_bitset, diff --git a/cpp/arcticdb/column_store/column_data.hpp b/cpp/arcticdb/column_store/column_data.hpp index 5a1955e2a7..67d96f5273 100644 --- a/cpp/arcticdb/column_store/column_data.hpp +++ b/cpp/arcticdb/column_store/column_data.hpp @@ -258,7 +258,7 @@ struct ColumnData { return data_.ptr_ == other.data_.ptr_; } - base_type::reference dereference() const { + typename base_type::reference dereference() const { if constexpr (iterator_type == IteratorType::ENUMERATED) { return data_; } else { @@ -270,7 +270,7 @@ struct ColumnData { ColumnData* parent_{nullptr}; std::optional> opt_block_{std::nullopt}; std::size_t remaining_values_in_block_{0}; - base_type::value_type data_; + typename base_type::value_type data_; }; ColumnData( diff --git a/cpp/arcticdb/entity/stream_descriptor.hpp b/cpp/arcticdb/entity/stream_descriptor.hpp index d6812dc203..917e6751b9 100644 --- a/cpp/arcticdb/entity/stream_descriptor.hpp +++ b/cpp/arcticdb/entity/stream_descriptor.hpp @@ -272,9 +272,13 @@ StreamDescriptor index_descriptor(const StreamId& stream_id, IndexType, const Ra } template -StreamDescriptor index_descriptor(StreamId stream_id, IndexType index_type, - std::initializer_list fields) { - return index_descriptor(stream_id, index_type, folly::gen::from(fields) | folly::gen::as()); +StreamDescriptor index_descriptor(StreamId stream_id, IndexType index_type, std::initializer_list fields) { + std::vector fields_vec; + fields_vec.reserve(fields.size()); + for(const auto& field : fields) + fields_vec.push_back(field); + + return index_descriptor(stream_id, index_type, fields_vec); } template diff --git a/cpp/arcticdb/log/log.cpp b/cpp/arcticdb/log/log.cpp index 205e3a9129..b4352b181f 100644 --- a/cpp/arcticdb/log/log.cpp +++ b/cpp/arcticdb/log/log.cpp @@ -33,16 +33,12 @@ namespace arcticdb::log { static const char* DefaultLogPattern = "%Y%m%d %H:%M:%S.%f %t %L %n | %v"; - namespace { - std::shared_ptr loggers_instance_; - std::once_flag loggers_init_flag_; -} - - -struct Loggers::Impl -{ +std::shared_ptr loggers_instance_; +std::once_flag loggers_init_flag_; +} // namespace +struct Loggers::Impl { std::mutex config_mutex_; std::unordered_map sink_by_id_; std::unique_ptr unconfigured_ = std::make_unique("arcticdb", @@ -70,7 +66,6 @@ struct Loggers::Impl spdlog::logger& logger_ref(std::unique_ptr& src); }; - constexpr auto get_default_log_level() { return spdlog::level::info; } @@ -128,15 +123,13 @@ namespace fs = std::filesystem; using SinkConf = arcticdb::proto::logger::SinkConfig; Loggers::Loggers() - : impl_(std::make_unique()) -{ + : impl_(std::make_unique()) { impl_->unconfigured_->set_level(get_default_log_level()); } Loggers::~Loggers() = default; -Loggers& Loggers::instance() -{ +Loggers& Loggers::instance() { std::call_once(loggers_init_flag_, &Loggers::init); return *loggers_instance_; } @@ -204,7 +197,6 @@ void Loggers::flush_all() { snapshot().flush(); } - void Loggers::destroy_instance() { loggers_instance_.reset(); } @@ -213,7 +205,6 @@ void Loggers::init() { loggers_instance_ = std::make_shared(); } - namespace { std::string make_parent_dir(const std::string &p_str, std::string_view def_p_str) { fs::path p; @@ -236,7 +227,6 @@ spdlog::logger& Loggers::Impl::logger_ref(std::unique_ptr& src) return *unconfigured_; } - bool Loggers::configure(const arcticdb::proto::logger::LoggersConfig &conf, bool force) { auto lock = std::scoped_lock(impl_->config_mutex_); if (!force && impl_->root_) @@ -320,7 +310,6 @@ bool Loggers::configure(const arcticdb::proto::logger::LoggersConfig &conf, bool check_and_configure("symbol", "root", impl_->symbol_); check_and_configure("snapshot", "root", impl_->snapshot_); - if (auto flush_sec = util::as_opt(conf.flush_interval_seconds()).value_or(1); flush_sec != 0) { impl_->periodic_worker_.emplace( [loggers = std::weak_ptr(loggers_instance_)]() { @@ -332,7 +321,6 @@ bool Loggers::configure(const arcticdb::proto::logger::LoggersConfig &conf, bool return true; } - void Loggers::Impl::configure_logger( const arcticdb::proto::logger::LoggerConfig &conf, const std::string &name, @@ -354,18 +342,15 @@ void Loggers::Impl::configure_logger( logger = std::make_unique(fq_name, sink_ptrs.begin(), sink_ptrs.end()); } - if (!conf.pattern().empty()) { + if (!conf.pattern().empty()) logger->set_pattern(conf.pattern()); - } - else { + else logger->set_pattern(DefaultLogPattern); - } - if (conf.level() != 0) { + if (conf.level() != 0) logger->set_level(static_cast(conf.level() - 1)); - } else { + else logger->set_level(get_default_log_level()); - } } } diff --git a/cpp/arcticdb/pipeline/index_writer.hpp b/cpp/arcticdb/pipeline/index_writer.hpp index 00d51b465e..ce8e32e8c4 100644 --- a/cpp/arcticdb/pipeline/index_writer.hpp +++ b/cpp/arcticdb/pipeline/index_writer.hpp @@ -17,7 +17,7 @@ namespace arcticdb::pipelines::index { // TODO: change the name - something like KeysSegmentWriter or KeyAggragator or better -template +template class IndexWriter { // All index segments are row-count indexed in the sense that the keys are // already ordered - they don't need an additional index diff --git a/cpp/arcticdb/pipeline/input_tensor_frame.hpp b/cpp/arcticdb/pipeline/input_tensor_frame.hpp index 69eef7a2b8..ad4efa1788 100644 --- a/cpp/arcticdb/pipeline/input_tensor_frame.hpp +++ b/cpp/arcticdb/pipeline/input_tensor_frame.hpp @@ -18,18 +18,6 @@ namespace arcticdb::pipelines { using namespace arcticdb::entity; -/// @TODO Move to a separate "util" header -template -concept is_any_of = (std::same_as || ...); - -template -concept ValidIndex = is_any_of< - std::remove_cvref_t>>, - stream::TimeseriesIndex, - stream::RowCountIndex, - stream::TableIndex, - stream::EmptyIndex>; - struct InputTensorFrame { InputTensorFrame() : diff --git a/cpp/arcticdb/processing/clause.hpp b/cpp/arcticdb/processing/clause.hpp index d646740fe8..d6b6e2841d 100644 --- a/cpp/arcticdb/processing/clause.hpp +++ b/cpp/arcticdb/processing/clause.hpp @@ -351,6 +351,13 @@ inline StreamDescriptor empty_descriptor(arcticdb::proto::descriptors::IndexDesc } struct NamedAggregator { + NamedAggregator(const std::string& s, const std::string& t, const std::string& v) : + aggregation_operator_(s), + input_column_name_(t), + output_column_name_(v){ + + } + std::string aggregation_operator_; std::string input_column_name_; std::string output_column_name_; diff --git a/cpp/arcticdb/processing/operation_dispatch.cpp b/cpp/arcticdb/processing/operation_dispatch.cpp index 3dd0558732..3924520219 100644 --- a/cpp/arcticdb/processing/operation_dispatch.cpp +++ b/cpp/arcticdb/processing/operation_dispatch.cpp @@ -50,7 +50,7 @@ VariantData transform_to_bitset(const VariantData& data) { details::visit_type(column_with_strings.column_->type().data_type(), [&column_with_strings, &output_bitset](auto col_tag) { using type_info = ScalarTypeInfo; if constexpr (is_bool_type(type_info::data_type)) { - Column::transform(*column_with_strings.column_, output_bitset, false, [](auto input_value) -> bool { + Column::transform_to_bitset(*column_with_strings.column_, output_bitset, false, [](auto input_value) -> bool { return input_value; }); } else { diff --git a/cpp/arcticdb/processing/operation_dispatch_binary.hpp b/cpp/arcticdb/processing/operation_dispatch_binary.hpp index fa047931a5..f29440949a 100644 --- a/cpp/arcticdb/processing/operation_dispatch_binary.hpp +++ b/cpp/arcticdb/processing/operation_dispatch_binary.hpp @@ -47,15 +47,15 @@ VariantData visit_binary_boolean(const VariantData& left, const VariantData& rig template VariantData binary_membership(const ColumnWithStrings& column_with_strings, ValueSet& value_set, Func&& func) { if (is_empty_type(column_with_strings.column_->type().data_type())) { - if constexpr(std::is_same_v) { + if constexpr(std::is_same_v, IsInOperator>) { return EmptyResult{}; - } else if constexpr(std::is_same_v) { + } else if constexpr(std::is_same_v, IsNotInOperator>) { return FullResult{}; } } // If the value set is empty, we can short-circuit if (value_set.empty()) { - if constexpr(std::is_same_v) { + if constexpr(std::is_same_v, IsNotInOperator>) { return FullResult{}; } else { return EmptyResult{}; @@ -63,7 +63,7 @@ VariantData binary_membership(const ColumnWithStrings& column_with_strings, Valu } util::BitSet output_bitset; - constexpr auto sparse_missing_value_output = std::is_same_v; + constexpr auto sparse_missing_value_output = std::is_same_v, IsNotInOperator>; details::visit_type(column_with_strings.column_->type().data_type(),[&] (auto col_tag) { using col_type_info = ScalarTypeInfo; details::visit_type(value_set.base_type().data_type(), [&] (auto val_set_tag) { @@ -79,7 +79,7 @@ VariantData binary_membership(const ColumnWithStrings& column_with_strings, Valu typed_value_set = value_set.get_set(); } auto offset_set = column_with_strings.string_pool_->get_offsets_for_column(typed_value_set, *column_with_strings.column_); - Column::transform( + Column::transform_to_bitset( *column_with_strings.column_, output_bitset, sparse_missing_value_output, @@ -92,7 +92,7 @@ VariantData binary_membership(const ColumnWithStrings& column_with_strings, Valu } else if constexpr (is_numeric_type(col_type_info::data_type) && is_numeric_type(val_set_type_info::data_type)) { using WideType = typename type_arithmetic_promoted_type>::type; auto typed_value_set = value_set.get_set(); - Column::transform( + Column::transform_to_bitset( *column_with_strings.column_, output_bitset, sparse_missing_value_output, @@ -123,7 +123,7 @@ VariantData visit_binary_membership(const VariantData &left, const VariantData & return std::visit(util::overload { [&] (const ColumnWithStrings& l, const std::shared_ptr& r) ->VariantData { - return transform_to_placeholder(binary_membership(l, *r, std::forward(func))); + return transform_to_placeholder(binary_membership(l, *r, std::forward(func))); }, [](const auto &, const auto&) -> VariantData { util::raise_rte("Binary membership operations must be Column/ValueSet"); @@ -137,7 +137,7 @@ VariantData binary_comparator(const ColumnWithStrings& left, const ColumnWithStr return EmptyResult{}; } util::BitSet output_bitset; - constexpr auto sparse_missing_value_output = std::is_same_v; + constexpr auto sparse_missing_value_output = std::is_same_v, NotEqualsOperator>; details::visit_type(left.column_->type().data_type(), [&](auto left_tag) { using left_type_info = ScalarTypeInfo; @@ -185,7 +185,7 @@ VariantData binary_comparator(const ColumnWithStrings& column_with_strings, cons return EmptyResult{}; } util::BitSet output_bitset; - constexpr auto sparse_missing_value_output = std::is_same_v; + constexpr auto sparse_missing_value_output = std::is_same_v, NotEqualsOperator>; details::visit_type(column_with_strings.column_->type().data_type(), [&](auto col_tag) { using col_type_info = ScalarTypeInfo; @@ -206,7 +206,7 @@ VariantData binary_comparator(const ColumnWithStrings& column_with_strings, cons value_string = std::string(*val.str_data(), val.len()); } auto value_offset = column_with_strings.string_pool_->get_offset_for_column(value_string, *column_with_strings.column_); - Column::transform( + Column::transform_to_bitset( *column_with_strings.column_, output_bitset, sparse_missing_value_output, @@ -224,7 +224,7 @@ VariantData binary_comparator(const ColumnWithStrings& column_with_strings, cons typename arcticdb::Comparable, typename arcticdb::Comparable>; auto value = static_cast(*reinterpret_cast(val.data_)); - Column::transform( + Column::transform_to_bitset( *column_with_strings.column_, output_bitset, sparse_missing_value_output, diff --git a/cpp/arcticdb/processing/operation_dispatch_unary.hpp b/cpp/arcticdb/processing/operation_dispatch_unary.hpp index c8018d7ba6..0f1bc821b4 100644 --- a/cpp/arcticdb/processing/operation_dispatch_unary.hpp +++ b/cpp/arcticdb/processing/operation_dispatch_unary.hpp @@ -102,9 +102,9 @@ VariantData visit_unary_operator(const VariantData& left, Func&& func) { template VariantData unary_comparator(const Column& col, Func&& func) { if (is_empty_type(col.type().data_type()) || is_integer_type(col.type().data_type())) { - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v, IsNullOperator>) { return is_empty_type(col.type().data_type()) ? VariantData(FullResult{}) : VariantData(EmptyResult{}); - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v, NotNullOperator>) { return is_empty_type(col.type().data_type()) ? VariantData(EmptyResult{}) : VariantData(FullResult{}); } else { internal::raise("Unexpected operator passed to unary_comparator"); @@ -112,10 +112,10 @@ VariantData unary_comparator(const Column& col, Func&& func) { } util::BitSet output_bitset; - constexpr auto sparse_missing_value_output = std::is_same_v; + constexpr auto sparse_missing_value_output = std::is_same_v, IsNullOperator>; details::visit_type(col.type().data_type(), [&](auto col_tag) { using type_info = ScalarTypeInfo; - Column::transform(col, output_bitset, sparse_missing_value_output, [&func](auto input_value) -> bool { + Column::transform_to_bitset(col, output_bitset, sparse_missing_value_output, [&func](auto input_value) -> bool { if constexpr (is_floating_point_type(type_info::data_type)) { return func.apply(input_value); } else if constexpr (is_sequence_type(type_info::data_type)) { @@ -134,15 +134,15 @@ template VariantData visit_unary_comparator(const VariantData& left, Func&& func) { return std::visit(util::overload{ [&] (const ColumnWithStrings& l) -> VariantData { - return transform_to_placeholder(unary_comparator(*(l.column_), std::forward(func))); + return transform_to_placeholder(unary_comparator(*(l.column_), std::forward(func))); }, [] (EmptyResult) -> VariantData { - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v, IsNullOperator>) { return FullResult{}; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v, NotNullOperator>) { return EmptyResult{}; } else { - internal::raise("Unexpected operator passed to unary_comparator"); + internal::raise("Unexpected operator passed to visit unary_comparator"); } }, [](const auto&) -> VariantData { diff --git a/cpp/arcticdb/processing/test/benchmark_clause.cpp b/cpp/arcticdb/processing/test/benchmark_clause.cpp index 616ee3253e..be8d570388 100644 --- a/cpp/arcticdb/processing/test/benchmark_clause.cpp +++ b/cpp/arcticdb/processing/test/benchmark_clause.cpp @@ -89,7 +89,6 @@ static void BM_merge_ordered(benchmark::State& state){ } template -requires std::integral void BM_hash_grouping_int(benchmark::State& state) { auto num_rows = state.range(0); auto num_unique_values = state.range(1); diff --git a/cpp/arcticdb/storage/coalesced/multi_segment_utils.hpp b/cpp/arcticdb/storage/coalesced/multi_segment_utils.hpp index e0923500f7..c629c1ac5f 100644 --- a/cpp/arcticdb/storage/coalesced/multi_segment_utils.hpp +++ b/cpp/arcticdb/storage/coalesced/multi_segment_utils.hpp @@ -22,7 +22,7 @@ uint64_t get_symbol_prefix(const entity::StreamId& stream_id) { constexpr size_t begin = sizeof(InternalType) - sizeof(StorageType); StorageType data{}; util::variant_match(stream_id, - [&data] (const entity::StringId& string_id) { + [&data, begin, end] (const entity::StringId& string_id) { auto* target = reinterpret_cast(&data); for(size_t p = begin, i = 0; p < end && i < string_id.size(); ++p, ++i) { const auto c = string_id[i]; diff --git a/cpp/arcticdb/storage/lmdb/lmdb.hpp b/cpp/arcticdb/storage/lmdb/lmdb.hpp new file mode 100644 index 0000000000..7e3ba325b5 --- /dev/null +++ b/cpp/arcticdb/storage/lmdb/lmdb.hpp @@ -0,0 +1,14 @@ +#pragma once + +// LMDB++ is using `std::is_pod` in `lmdb++.h`, which is deprecated as of C++20. +// See: https://github.com/drycpp/lmdbxx/blob/0b43ca87d8cfabba392dfe884eb1edb83874de02/lmdb%2B%2B.h#L1068 +// See: https://en.cppreference.com/w/cpp/types/is_pod +// This suppresses the warning. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#ifdef ARCTICDB_USING_CONDA +#include +#else +#include +#endif +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/cpp/arcticdb/storage/lmdb/lmdb_client_wrapper.hpp b/cpp/arcticdb/storage/lmdb/lmdb_client_wrapper.hpp index 9b7ca48397..e1860f3bf1 100644 --- a/cpp/arcticdb/storage/lmdb/lmdb_client_wrapper.hpp +++ b/cpp/arcticdb/storage/lmdb/lmdb_client_wrapper.hpp @@ -12,18 +12,10 @@ #include #include -// LMDB++ is using `std::is_pod` in `lmdb++.h`, which is deprecated as of C++20. -// See: https://github.com/drycpp/lmdbxx/blob/0b43ca87d8cfabba392dfe884eb1edb83874de02/lmdb%2B%2B.h#L1068 -// See: https://en.cppreference.com/w/cpp/types/is_pod -// This suppresses the warning. -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#ifdef ARCTICDB_USING_CONDA -#include -#else -#include -#endif -#pragma GCC diagnostic pop +namespace lmdb { +class txn; +class dbi; +} namespace arcticdb::storage::lmdb { diff --git a/cpp/arcticdb/storage/lmdb/lmdb_mock_client.cpp b/cpp/arcticdb/storage/lmdb/lmdb_mock_client.cpp index 879089797c..f95ff09c0e 100644 --- a/cpp/arcticdb/storage/lmdb/lmdb_mock_client.cpp +++ b/cpp/arcticdb/storage/lmdb/lmdb_mock_client.cpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace arcticdb::storage::lmdb { diff --git a/cpp/arcticdb/storage/lmdb/lmdb_real_client.cpp b/cpp/arcticdb/storage/lmdb/lmdb_real_client.cpp index 4eb35fcd92..180650ad82 100644 --- a/cpp/arcticdb/storage/lmdb/lmdb_real_client.cpp +++ b/cpp/arcticdb/storage/lmdb/lmdb_real_client.cpp @@ -11,7 +11,7 @@ #include #include #include - +#include namespace arcticdb::storage::lmdb { diff --git a/cpp/arcticdb/storage/lmdb/lmdb_storage.cpp b/cpp/arcticdb/storage/lmdb/lmdb_storage.cpp index fa1f3c568d..fb441b1017 100644 --- a/cpp/arcticdb/storage/lmdb/lmdb_storage.cpp +++ b/cpp/arcticdb/storage/lmdb/lmdb_storage.cpp @@ -18,14 +18,13 @@ #include #include #include -#include #include #include #include -#include #include #include +#include namespace arcticdb::storage::lmdb { @@ -49,6 +48,15 @@ void raise_lmdb_exception(const ::lmdb::error& e) { raise(fmt::format("Unexpected LMDB Error: LMDBError#{}: {}", error_code, e.what())); } +::lmdb::env& LmdbStorage::env() { + if (!env_) { + raise("Unexpected LMDB Error: Invalid operation: LMDB environment has been removed. " + "Possibly because the library has been deleted"); + } + return *env_; +} + + void LmdbStorage::do_write_internal(Composite&& kvs, ::lmdb::txn& txn) { auto fmt_db = [](auto &&kv) { return kv.key_type(); }; @@ -56,7 +64,7 @@ void LmdbStorage::do_write_internal(Composite&& kvs, ::lmdb::txn auto db_name = fmt::format("{}", group.key()); ARCTICDB_SUBSAMPLE(LmdbStorageOpenDb, 0) - ::lmdb::dbi& dbi = dbi_by_key_type_.at(db_name); + ::lmdb::dbi& dbi = *dbi_by_key_type_.at(db_name); ARCTICDB_SUBSAMPLE(LmdbStorageWriteValues, 0) for (auto &kv : group.values()) { @@ -117,7 +125,7 @@ void LmdbStorage::do_read(Composite&& ks, const ReadVisitor& visitor (fg::from(ks.as_range()) | fg::move | fg::groupBy(fmt_db)).foreach([&](auto &&group) { auto db_name = fmt::format("{}", group.key()); ARCTICDB_SUBSAMPLE(LmdbStorageOpenDb, 0) - ::lmdb::dbi& dbi = dbi_by_key_type_.at(db_name); + ::lmdb::dbi& dbi = *dbi_by_key_type_.at(db_name); for (auto &k : group.values()) { auto stored_key = to_serialized_key(k); try { @@ -155,7 +163,7 @@ bool LmdbStorage::do_key_exists(const VariantKey&key) { ARCTICDB_SUBSAMPLE(LmdbStorageOpenDb, 0) try { - ::lmdb::dbi& dbi = dbi_by_key_type_.at(db_name); + ::lmdb::dbi& dbi = *dbi_by_key_type_.at(db_name); auto stored_key = to_serialized_key(key); return lmdb_client_->exists(db_name, stored_key, txn, dbi); } catch ([[maybe_unused]] const ::lmdb::not_found_error &ex) { @@ -176,7 +184,7 @@ std::vector LmdbStorage::do_remove_internal(Composite&& ARCTICDB_SUBSAMPLE(LmdbStorageOpenDb, 0) try { // If no key of this type has been written before, this can fail - ::lmdb::dbi& dbi = dbi_by_key_type_.at(db_name); + ::lmdb::dbi& dbi = *dbi_by_key_type_.at(db_name); for (auto &k : group.values()) { auto stored_key = to_serialized_key(k); @@ -230,7 +238,7 @@ bool LmdbStorage::do_fast_delete() { auto db_name = fmt::format("{}", key_type); ARCTICDB_SUBSAMPLE(LmdbStorageOpenDb, 0) ARCTICDB_DEBUG(log::storage(), "dropping {}", db_name); - ::lmdb::dbi& dbi = dbi_by_key_type_.at(db_name); + ::lmdb::dbi& dbi = *dbi_by_key_type_.at(db_name); try { ::lmdb::dbi_drop(dtxn, dbi); } catch (const ::lmdb::error& ex) { @@ -246,7 +254,7 @@ void LmdbStorage::do_iterate_type(KeyType key_type, const IterateTypeVisitor& vi ARCTICDB_SAMPLE(LmdbStorageItType, 0); auto txn = ::lmdb::txn::begin(env(), nullptr, MDB_RDONLY); // scoped abort on std::string type_db = fmt::format("{}", key_type); - ::lmdb::dbi& dbi = dbi_by_key_type_.at(type_db); + ::lmdb::dbi& dbi = *dbi_by_key_type_.at(type_db); try { auto keys = lmdb_client_->list(type_db, prefix, txn, dbi, key_type); @@ -332,7 +340,7 @@ LmdbStorage::LmdbStorage(const LibraryPath &library_path, OpenMode mode, const C } write_mutex_ = std::make_unique(); env_ = std::make_unique<::lmdb::env>(::lmdb::env::create(conf.flags())); - dbi_by_key_type_ = std::unordered_map{}; + dbi_by_key_type_ = std::unordered_map>{}; fs::path root_path = conf.path().c_str(); auto lib_path_str = library_path.to_delim_path(fs::path::preferred_separator); @@ -381,7 +389,7 @@ LmdbStorage::LmdbStorage(const LibraryPath &library_path, OpenMode mode, const C arcticdb::entity::foreach_key_type([&txn, this](KeyType &&key_type) { std::string db_name = fmt::format("{}", key_type); ::lmdb::dbi dbi = ::lmdb::dbi::open(txn, db_name.data(), MDB_CREATE); - dbi_by_key_type_.insert(std::make_pair(std::move(db_name), std::move(dbi))); + dbi_by_key_type_.insert(std::make_pair(std::move(db_name), std::make_unique<::lmdb::dbi>(std::move(dbi)))); }); } catch (const ::lmdb::error& ex) { raise_lmdb_exception(ex); diff --git a/cpp/arcticdb/storage/lmdb/lmdb_storage.hpp b/cpp/arcticdb/storage/lmdb/lmdb_storage.hpp index 5629738084..6824a20aca 100644 --- a/cpp/arcticdb/storage/lmdb/lmdb_storage.hpp +++ b/cpp/arcticdb/storage/lmdb/lmdb_storage.hpp @@ -20,6 +20,11 @@ namespace fs = std::filesystem; +namespace lmdb { +class env; +class dbi; +} + namespace arcticdb::storage::lmdb { class LmdbStorage final : public Storage { @@ -54,13 +59,7 @@ class LmdbStorage final : public Storage { bool do_is_path_valid(const std::string_view path) const final; - ::lmdb::env& env() { - if (!env_) { - raise("Unexpected LMDB Error: Invalid operation: LMDB environment has been removed. " - "Possibly because the library has been deleted"); - } - return *env_; - } + ::lmdb::env& env(); std::string do_key_path(const VariantKey&) const final { return {}; }; @@ -72,7 +71,7 @@ class LmdbStorage final : public Storage { std::unique_ptr write_mutex_; std::unique_ptr<::lmdb::env> env_; - std::unordered_map dbi_by_key_type_; + std::unordered_map> dbi_by_key_type_; std::filesystem::path lib_dir_; diff --git a/cpp/arcticdb/storage/mongo/mongo_mock_client.cpp b/cpp/arcticdb/storage/mongo/mongo_mock_client.cpp index 605991d21b..cedf82baf8 100644 --- a/cpp/arcticdb/storage/mongo/mongo_mock_client.cpp +++ b/cpp/arcticdb/storage/mongo/mongo_mock_client.cpp @@ -5,7 +5,6 @@ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. */ -#include #include #include #include @@ -14,7 +13,6 @@ #include #include - namespace arcticdb::storage::mongo { std::string MockMongoClient::get_failure_trigger( @@ -45,7 +43,7 @@ MongoFailure get_failure(const std::string& message, StorageOperation operation, case StorageOperation::EXISTS: return create_failure(message, error_code); case StorageOperation::DELETE: - return create_failure(message, error_code); + [[fallthrough]]; case StorageOperation::DELETE_LOCAL: return create_failure(message, error_code); case StorageOperation::LIST: @@ -58,7 +56,7 @@ MongoFailure get_failure(const std::string& message, StorageOperation operation, std::optional has_failure_trigger( const MongoKey& key, StorageOperation operation) { - auto key_id = key.doc_key.id_string(); + auto key_id = key.doc_key_.id_string(); auto failure_string_for_operation = "#Failure_" + operation_to_string(operation) + "_"; auto position = key_id.rfind(failure_string_for_operation); if (position == std::string::npos) { @@ -83,8 +81,8 @@ bool matches_prefix( const std::string& collection_name, const std::string& prefix) { - return key.database_name == database_name && key.collection_name == collection_name && - key.doc_key.id_string().find(prefix) == 0; + return key.database_name_ == database_name && key.collection_name_ == collection_name && + key.doc_key_.id_string().find(prefix) == 0; } void throw_if_exception(MongoFailure& failure) { @@ -109,7 +107,7 @@ bool MockMongoClient::write_segment( return false; } - mongo_contents.insert_or_assign(key, std::move(kv.segment())); + mongo_contents.insert_or_assign(std::move(key), std::move(kv.segment())); return true; } @@ -131,7 +129,7 @@ UpdateResult MockMongoClient::update_segment( return {0}; // upsert is false, don't update and return 0 as modified_count } - mongo_contents.insert_or_assign(key, std::move(kv.segment())); + mongo_contents.insert_or_assign(std::move(key), std::move(kv.segment())); return {key_found ? 1 : 0}; } @@ -151,7 +149,7 @@ std::optional MockMongoClient::read_segment( return std::nullopt; } - return KeySegmentPair(std::move(mongo_key.doc_key.key), std::move(it->second)); + return KeySegmentPair(std::move(mongo_key.doc_key_.key_), std::move(it->second)); } DeleteResult MockMongoClient::remove_keyvalue( @@ -203,7 +201,7 @@ std::vector MockMongoClient::list_keys( throw_if_exception(failure.value()); return {}; } - output.push_back(key.first.doc_key.key); + output.push_back(key.first.doc_key_.key_); } } @@ -216,7 +214,7 @@ void MockMongoClient::ensure_collection(std::string_view, std::string_view ) { void MockMongoClient::drop_collection(std::string database_name, std::string collection_name) { for (auto it = mongo_contents.begin(); it != mongo_contents.end(); ) { - if (it->first.database_name == database_name && it->first.collection_name == collection_name) { + if (it->first.database_name_ == database_name && it->first.collection_name_ == collection_name) { it = mongo_contents.erase(it); } else { ++it; diff --git a/cpp/arcticdb/storage/mongo/mongo_mock_client.hpp b/cpp/arcticdb/storage/mongo/mongo_mock_client.hpp index ec4ea72228..3c2ab02d69 100644 --- a/cpp/arcticdb/storage/mongo/mongo_mock_client.hpp +++ b/cpp/arcticdb/storage/mongo/mongo_mock_client.hpp @@ -15,27 +15,36 @@ namespace arcticdb::storage::mongo { struct MongoDocumentKey { - VariantKey key; + VariantKey key_; - std::string id_string() const { - return fmt::format("{}", variant_key_id(key)); + ARCTICDB_MOVE_ONLY_DEFAULT(MongoDocumentKey) + + explicit MongoDocumentKey(VariantKey&& key) : key_(std::move(key)) { + } + + [[nodiscard]] std::string id_string() const { + return fmt::format("{}", variant_key_id(key_)); } }; struct MongoKey { - std::string database_name; - std::string collection_name; - MongoDocumentKey doc_key; + std::string database_name_; + std::string collection_name_; + MongoDocumentKey doc_key_; + + ARCTICDB_MOVE_ONLY_DEFAULT(MongoKey) - MongoKey(const std::string& database_name, const std::string& collection_name, const VariantKey& key) : - database_name(database_name), collection_name(collection_name), doc_key(key) { } + MongoKey(std::string database_name, std::string collection_name, VariantKey key) : + database_name_(std::move(database_name)), + collection_name_(std::move(collection_name)), + doc_key_(std::move(key)) { } bool operator<(const MongoKey& other) const { - std::string id_string = doc_key.id_string(); - std::string other_id_string = other.doc_key.id_string(); + std::string id_string = doc_key_.id_string(); + std::string other_id_string = other.doc_key_.id_string(); - return std::tie(database_name, collection_name, id_string) < - std::tie(other.database_name, other.collection_name, other_id_string); + return std::tie(database_name_, collection_name_, id_string) < + std::tie(other.database_name_, other.collection_name_, other_id_string); } }; @@ -50,15 +59,17 @@ struct MongoFailure { return std::holds_alternative(failure); } - mongocxx::operation_exception get_exception() const { + [[nodiscard]] mongocxx::operation_exception get_exception() const { return std::get(failure); } - }; class MockMongoClient : public MongoClientWrapper { - public: + MockMongoClient() = default; + + ARCTICDB_MOVE_ONLY_DEFAULT(MockMongoClient) + static std::string get_failure_trigger( const std::string& key, StorageOperation operation_to_fail, diff --git a/cpp/arcticdb/storage/python_bindings.cpp b/cpp/arcticdb/storage/python_bindings.cpp index c573852554..c2dfaf53ee 100644 --- a/cpp/arcticdb/storage/python_bindings.cpp +++ b/cpp/arcticdb/storage/python_bindings.cpp @@ -15,7 +15,6 @@ #include #include -#include #include #include #include diff --git a/cpp/arcticdb/storage/test/test_storage_exceptions.cpp b/cpp/arcticdb/storage/test/test_storage_exceptions.cpp index a7d422ec68..2dbc217bd5 100644 --- a/cpp/arcticdb/storage/test/test_storage_exceptions.cpp +++ b/cpp/arcticdb/storage/test/test_storage_exceptions.cpp @@ -26,6 +26,7 @@ #include #include +#include using namespace arcticdb; using namespace storage; diff --git a/cpp/arcticdb/util/cxx17_concepts.hpp b/cpp/arcticdb/util/cxx17_concepts.hpp new file mode 100644 index 0000000000..6fb9fc2a96 --- /dev/null +++ b/cpp/arcticdb/util/cxx17_concepts.hpp @@ -0,0 +1,18 @@ +#include +#include + +namespace arcticdb { + + template + using is_unary_predicate = std::is_invocable_r; + + template + constexpr bool is_unary_predicate_v = is_unary_predicate::value; + + template + using is_binary_predicate = std::is_invocable_r; + + template + constexpr bool is_binary_predicate_v = is_binary_predicate::value; + +} \ No newline at end of file diff --git a/cpp/arcticdb/util/format_date.hpp b/cpp/arcticdb/util/format_date.hpp index d02f171118..4e5b0fce04 100644 --- a/cpp/arcticdb/util/format_date.hpp +++ b/cpp/arcticdb/util/format_date.hpp @@ -19,7 +19,7 @@ using TimePoint = std::chrono::time_point; inline std::string format_timestamp(entity::timestamp ts) { std::stringstream ss; -#if defined(_WIN32) or defined(__APPLE__) +#if defined(_WIN32) || defined(__APPLE__) auto time_point = Clock::time_point(std::chrono::duration_cast(std::chrono::nanoseconds(ts))); #else auto time_point = Clock::time_point(std::chrono::nanoseconds(ts)); diff --git a/cpp/arcticdb/util/offset_string.cpp b/cpp/arcticdb/util/offset_string.cpp index 963b65892b..1df858d934 100644 --- a/cpp/arcticdb/util/offset_string.cpp +++ b/cpp/arcticdb/util/offset_string.cpp @@ -24,9 +24,5 @@ OffsetString::offset_t OffsetString::offset() const { } // Given a set of string pool offsets, removes any that represent None or NaN -void remove_nones_and_nans(ankerl::unordered_dense::set& offsets) { - offsets.erase(not_a_string()); - offsets.erase(nan_placeholder()); -} } //namespace arcticdb diff --git a/cpp/arcticdb/util/offset_string.hpp b/cpp/arcticdb/util/offset_string.hpp index f7d37307a0..03db23f168 100644 --- a/cpp/arcticdb/util/offset_string.hpp +++ b/cpp/arcticdb/util/offset_string.hpp @@ -52,8 +52,10 @@ constexpr bool is_a_string(OffsetString::offset_t offset) { } // Given a set of string pool offsets, removes any that represent None or NaN -void remove_nones_and_nans(ankerl::unordered_dense::set& offsets); - +inline void remove_nones_and_nans(ankerl::unordered_dense::set& offsets) { + offsets.erase(not_a_string()); + offsets.erase(nan_placeholder()); +} template inline PyObject* create_py_nan(LockPtrType& lock) { diff --git a/cpp/arcticdb/version/local_versioned_engine.cpp b/cpp/arcticdb/version/local_versioned_engine.cpp index 0fd631a72a..2b1a413dc9 100644 --- a/cpp/arcticdb/version/local_versioned_engine.cpp +++ b/cpp/arcticdb/version/local_versioned_engine.cpp @@ -867,7 +867,7 @@ folly::Future LocalVersionedEngine::delete_trees_responsibly( return true; } get_matching_prev_and_next_versions(entry, key.version_id(), // Check 2) - [](ARCTICDB_UNUSED auto& matching) {}, + [](const AtomKey&) {}, [&check, ¬_to_delete](auto& prev) { if (check.prev_version) not_to_delete.insert(prev);}, [&check, ¬_to_delete](auto& next) { if (check.next_version) not_to_delete.insert(next);}, [v=key.version_id()](const AtomKeyImpl& key, const std::shared_ptr& entry) { diff --git a/python/tests/integration/test_import.py b/python/tests/integration/test_import.py new file mode 100644 index 0000000000..1b6feb5889 --- /dev/null +++ b/python/tests/integration/test_import.py @@ -0,0 +1,5 @@ +#import ray +#import arcticdb + +def test_thing(lmdb_version_store): + lmdb_version_store.write("Hello", 23) \ No newline at end of file