From 0c33c606b1d3502deed8eb8f8adf1eddf1cc17e1 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 7 Sep 2022 19:35:31 -0400 Subject: [PATCH] Updated C++ client code, tests, and examples. (#2818) * Updated C++ client code, tests, and examples. * Move third-party flatbuffers to a different directory; write a README.md file; make it clear how I patched flatbuffers. --- cpp-client/deephaven/client/CMakeLists.txt | 26 +- .../client/arrowutil/arrow_value_converter.h | 30 + .../client/arrowutil/arrow_visitors.h | 52 +- .../client/immerutil/abstract_flex_vector.h | 128 +++- .../client/immerutil/immer_column_source.h | 183 +++-- .../deephaven/client/impl/table_handle_impl.h | 3 +- .../client/subscription/batch_parser.h | 2 +- .../client/subscription/classic_table_state.h | 2 + .../client/subscription/immer_table_state.h | 5 +- .../client/subscription/subscribe_thread.h | 3 +- .../client/subscription/update_processor.h | 10 +- .../private/deephaven/client/utility/misc.h | 2 + .../public/deephaven/client/chunk/chunk.h | 19 +- .../deephaven/client/chunk/chunk_filler.h | 3 +- .../include/public/deephaven/client/client.h | 7 +- .../client/column/array_column_source.h | 325 +++++++++ .../deephaven/client/column/column_source.h | 178 +---- .../deephaven/client/container/row_sequence.h | 18 +- .../public/deephaven/client/table/table.h | 54 +- .../include/public/deephaven/client/ticking.h | 112 +-- .../include/public/deephaven/client/types.h | 35 +- .../public/deephaven/client/utility/utility.h | 23 +- .../client/src/chunk/chunk_filler.cc | 75 +- .../deephaven/client/src/chunk/chunk_maker.cc | 9 + cpp-client/deephaven/client/src/client.cc | 4 +- .../client/src/column/array_column_source.cc | 15 + .../client/src/container/row_sequence.cc | 131 ++-- .../client/src/impl/table_handle_impl.cc | 4 +- .../client/src/subscription/batch_parser.cc | 13 +- .../src/subscription/classic_table_state.cc | 80 ++- .../src/subscription/immer_table_state.cc | 40 +- .../src/subscription/shift_processor.cc | 8 +- .../src/subscription/subscribe_thread.cc | 15 +- .../src/subscription/update_processor.cc | 276 +++---- .../deephaven/client/src/table/table.cc | 360 ++++++++++ cpp-client/deephaven/client/src/ticking.cc | 46 +- cpp-client/deephaven/client/src/types.cc | 2 +- .../deephaven/client/src/utility/utility.cc | 7 + .../third_party/flatbuffers/LICENSE.txt | 7 - .../client/third_party/flatbuffers/README.md | 13 + .../{ => include/flatbuffers}/allocator.h | 0 .../{ => include/flatbuffers}/array.h | 0 .../{ => include/flatbuffers}/base.h | 0 .../{ => include/flatbuffers}/buffer.h | 0 .../{ => include/flatbuffers}/buffer_ref.h | 0 .../flatbuffers}/default_allocator.h | 0 .../flatbuffers}/detached_buffer.h | 0 .../flatbuffers}/flatbuffer_builder.h | 0 .../{ => include/flatbuffers}/flatbuffers.h | 0 .../{ => include/flatbuffers}/stl_emulation.h | 0 .../{ => include/flatbuffers}/string.h | 0 .../{ => include/flatbuffers}/struct.h | 0 .../{ => include/flatbuffers}/table.h | 0 .../{ => include/flatbuffers}/util.h | 0 .../{ => include/flatbuffers}/vector.h | 0 .../flatbuffers}/vector_downward.h | 0 .../{ => include/flatbuffers}/verifier.h | 0 .../client/third_party/flatbuffers/patch.001 | 350 +++++++++ cpp-client/tests/CMakeLists.txt | 31 +- .../{add_drop_example.cc => add_drop_test.cc} | 15 +- ...gregates_example.cc => aggregates_test.cc} | 34 +- .../{filter_example.cc => filter_test.cc} | 15 +- ..._tail_example.cc => head_and_tail_test.cc} | 14 +- .../tests/{join_example.cc => join_test.cc} | 14 +- .../{lastby_example.cc => lastby_test.cc} | 18 +- cpp-client/tests/main.cc | 2 +- ...tables_example.cc => merge_tables_test.cc} | 10 +- ...new_table_example.cc => new_table_test.cc} | 18 +- .../{select_example.cc => select_test.cc} | 21 +- .../tests/{sort_example.cc => sort_test.cc} | 16 +- ...ilter_example.cc => string_filter_test.cc} | 37 +- cpp-client/tests/test_util.cc | 12 +- cpp-client/tests/test_util.h | 22 +- cpp-client/tests/{ => third_party}/catch.hpp | 0 .../{ungroup_example.cc => ungroup_test.cc} | 15 +- ...lidation_example.cc => validation_test.cc} | 72 +- .../tests/{view_example.cc => view_test.cc} | 14 +- .../build-all-examples/CMakeLists.txt | 4 +- cpp-examples/cleanup/CMakeLists.txt | 6 - .../compare_approaches/CMakeLists.txt | 20 - cpp-examples/compare_approaches/main.cc | 671 ------------------ .../CMakeLists.txt | 6 - .../create_table_with_arrow_flight/main.cc | 2 +- .../CMakeLists.txt | 6 - cpp-examples/do_exchange/CMakeLists.txt | 19 - cpp-examples/do_exchange/main.cc | 556 --------------- cpp-examples/hello_world/CMakeLists.txt | 6 - cpp-examples/read_csv/CMakeLists.txt | 11 + cpp-examples/{readcsv => read_csv}/main.cc | 0 cpp-examples/read_csv/test.csv | 2 + .../CMakeLists.txt | 6 - cpp-examples/readcsv/CMakeLists.txt | 17 - 92 files changed, 2105 insertions(+), 2282 deletions(-) create mode 100644 cpp-client/deephaven/client/include/private/deephaven/client/arrowutil/arrow_value_converter.h create mode 100644 cpp-client/deephaven/client/include/public/deephaven/client/column/array_column_source.h create mode 100644 cpp-client/deephaven/client/src/column/array_column_source.cc delete mode 100644 cpp-client/deephaven/client/third_party/flatbuffers/LICENSE.txt create mode 100644 cpp-client/deephaven/client/third_party/flatbuffers/README.md rename cpp-client/deephaven/client/third_party/flatbuffers/{ => include/flatbuffers}/allocator.h (100%) rename cpp-client/deephaven/client/third_party/flatbuffers/{ => include/flatbuffers}/array.h (100%) rename cpp-client/deephaven/client/third_party/flatbuffers/{ => include/flatbuffers}/base.h (100%) rename cpp-client/deephaven/client/third_party/flatbuffers/{ => include/flatbuffers}/buffer.h (100%) rename cpp-client/deephaven/client/third_party/flatbuffers/{ => include/flatbuffers}/buffer_ref.h (100%) rename cpp-client/deephaven/client/third_party/flatbuffers/{ => include/flatbuffers}/default_allocator.h (100%) rename cpp-client/deephaven/client/third_party/flatbuffers/{ => include/flatbuffers}/detached_buffer.h (100%) rename cpp-client/deephaven/client/third_party/flatbuffers/{ => include/flatbuffers}/flatbuffer_builder.h (100%) rename cpp-client/deephaven/client/third_party/flatbuffers/{ => include/flatbuffers}/flatbuffers.h (100%) rename cpp-client/deephaven/client/third_party/flatbuffers/{ => include/flatbuffers}/stl_emulation.h (100%) rename cpp-client/deephaven/client/third_party/flatbuffers/{ => include/flatbuffers}/string.h (100%) rename cpp-client/deephaven/client/third_party/flatbuffers/{ => include/flatbuffers}/struct.h (100%) rename cpp-client/deephaven/client/third_party/flatbuffers/{ => include/flatbuffers}/table.h (100%) rename cpp-client/deephaven/client/third_party/flatbuffers/{ => include/flatbuffers}/util.h (100%) rename cpp-client/deephaven/client/third_party/flatbuffers/{ => include/flatbuffers}/vector.h (100%) rename cpp-client/deephaven/client/third_party/flatbuffers/{ => include/flatbuffers}/vector_downward.h (100%) rename cpp-client/deephaven/client/third_party/flatbuffers/{ => include/flatbuffers}/verifier.h (100%) create mode 100644 cpp-client/deephaven/client/third_party/flatbuffers/patch.001 rename cpp-client/tests/{add_drop_example.cc => add_drop_test.cc} (75%) rename cpp-client/tests/{aggregates_example.cc => aggregates_test.cc} (72%) rename cpp-client/tests/{filter_example.cc => filter_test.cc} (83%) rename cpp-client/tests/{head_and_tail_example.cc => head_and_tail_test.cc} (80%) rename cpp-client/tests/{join_example.cc => join_test.cc} (81%) rename cpp-client/tests/{lastby_example.cc => lastby_test.cc} (72%) rename cpp-client/tests/{merge_tables_example.cc => merge_tables_test.cc} (88%) rename cpp-client/tests/{new_table_example.cc => new_table_test.cc} (83%) rename cpp-client/tests/{select_example.cc => select_test.cc} (94%) rename cpp-client/tests/{sort_example.cc => sort_test.cc} (91%) rename cpp-client/tests/{string_filter_example.cc => string_filter_test.cc} (82%) rename cpp-client/tests/{ => third_party}/catch.hpp (100%) rename cpp-client/tests/{ungroup_example.cc => ungroup_test.cc} (76%) rename cpp-client/tests/{validation_example.cc => validation_test.cc} (58%) rename cpp-client/tests/{view_example.cc => view_test.cc} (81%) delete mode 100644 cpp-examples/compare_approaches/CMakeLists.txt delete mode 100644 cpp-examples/compare_approaches/main.cc delete mode 100644 cpp-examples/do_exchange/CMakeLists.txt delete mode 100644 cpp-examples/do_exchange/main.cc create mode 100644 cpp-examples/read_csv/CMakeLists.txt rename cpp-examples/{readcsv => read_csv}/main.cc (100%) create mode 100644 cpp-examples/read_csv/test.csv delete mode 100644 cpp-examples/readcsv/CMakeLists.txt diff --git a/cpp-client/deephaven/client/CMakeLists.txt b/cpp-client/deephaven/client/CMakeLists.txt index f581256e186..9567eabf0e2 100644 --- a/cpp-client/deephaven/client/CMakeLists.txt +++ b/cpp-client/deephaven/client/CMakeLists.txt @@ -44,6 +44,7 @@ set(ALL_FILES include/private/deephaven/client/impl/util.h include/private/deephaven/client/arrowutil/arrow_visitors.h + include/private/deephaven/client/arrowutil/arrow_value_converter.h src/columns.cc src/expressions.cc @@ -53,6 +54,7 @@ set(ALL_FILES src/chunk/chunk.cc src/chunk/chunk_filler.cc src/chunk/chunk_maker.cc + src/column/array_column_source.cc src/column/column_source.cc src/container/row_sequence.cc src/table/table.cc @@ -65,6 +67,7 @@ set(ALL_FILES include/public/deephaven/client/chunk/chunk.h include/public/deephaven/client/chunk/chunk_filler.h include/public/deephaven/client/chunk/chunk_maker.h + include/public/deephaven/client/column/array_column_source.h include/public/deephaven/client/column/column_source.h include/public/deephaven/client/container/row_sequence.h include/public/deephaven/client/table/table.h @@ -87,7 +90,6 @@ set(ALL_FILES src/subscription/subscribe_thread.cc src/subscription/update_processor.cc - src/immerutil/abstract_flex_vector.cc src/immerutil/immer_column_source.cc include/private/deephaven/client/immerutil/abstract_flex_vector.h @@ -108,6 +110,8 @@ set(ALL_FILES include/public/deephaven/client/utility/table_maker.h include/public/deephaven/client/utility/utility.h + flatbuf/deephaven/flatbuf/Barrage_generated.h + proto/deephaven/proto/application.grpc.pb.cc proto/deephaven/proto/application.grpc.pb.h proto/deephaven/proto/application.pb.cc @@ -141,7 +145,23 @@ set(ALL_FILES proto/deephaven/proto/ticket.pb.cc proto/deephaven/proto/ticket.pb.h - flatbuf/deephaven/flatbuf/Barrage_generated.h + third_party/flatbuffers/include/flatbuffers/allocator.h + third_party/flatbuffers/include/flatbuffers/array.h + third_party/flatbuffers/include/flatbuffers/base.h + third_party/flatbuffers/include/flatbuffers/buffer.h + third_party/flatbuffers/include/flatbuffers/buffer_ref.h + third_party/flatbuffers/include/flatbuffers/default_allocator.h + third_party/flatbuffers/include/flatbuffers/detached_buffer.h + third_party/flatbuffers/include/flatbuffers/flatbuffer_builder.h + third_party/flatbuffers/include/flatbuffers/flatbuffers.h + third_party/flatbuffers/include/flatbuffers/stl_emulation.h + third_party/flatbuffers/include/flatbuffers/string.h + third_party/flatbuffers/include/flatbuffers/struct.h + third_party/flatbuffers/include/flatbuffers/table.h + third_party/flatbuffers/include/flatbuffers/util.h + third_party/flatbuffers/include/flatbuffers/vector_downward.h + third_party/flatbuffers/include/flatbuffers/vector.h + third_party/flatbuffers/include/flatbuffers/verifier.h ) add_library(client ${ALL_FILES}) @@ -152,7 +172,7 @@ target_compile_options(client PRIVATE -Wall -Werror) target_include_directories(client PRIVATE ${Boost_INCLUDE_DIR}) target_include_directories(client PRIVATE include/private) -target_include_directories(client PRIVATE third_party) +target_include_directories(client PRIVATE third_party/flatbuffers/include) target_include_directories(client PUBLIC $) # Protos and flatbuf are doing their own thing. diff --git a/cpp-client/deephaven/client/include/private/deephaven/client/arrowutil/arrow_value_converter.h b/cpp-client/deephaven/client/include/private/deephaven/client/arrowutil/arrow_value_converter.h new file mode 100644 index 00000000000..3294db399f2 --- /dev/null +++ b/cpp-client/deephaven/client/include/private/deephaven/client/arrowutil/arrow_value_converter.h @@ -0,0 +1,30 @@ +/** + * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending + */ +#pragma once + +#include +#include +#include +#include + +namespace deephaven::client::arrowutil { +class ArrowValueConverter { +public: + // "convert" function for anything except string_view is the identity function. + template + static void convert(SRC src, DEST *dest) { + *dest = src; + } + + // "convert" function for string_view is std::string + static void convert(arrow::util::string_view sv, std::string *dest) { + dest->clear(); + dest->append(sv.data(), sv.size()); + } + + static void convert(int64_t src, deephaven::client::DateTime *dest) { + *dest = deephaven::client::DateTime::fromNanos(src); + } +}; +} // namespace deephaven::client::arrowutil diff --git a/cpp-client/deephaven/client/include/private/deephaven/client/arrowutil/arrow_visitors.h b/cpp-client/deephaven/client/include/private/deephaven/client/arrowutil/arrow_visitors.h index a280cc41727..927b662af00 100644 --- a/cpp-client/deephaven/client/include/private/deephaven/client/arrowutil/arrow_visitors.h +++ b/cpp-client/deephaven/client/include/private/deephaven/client/arrowutil/arrow_visitors.h @@ -4,6 +4,7 @@ #pragma once #include +#include "deephaven/client/types.h" namespace deephaven::client::arrowutil { @@ -33,21 +34,31 @@ class ArrowTypeVisitor final : public arrow::TypeVisitor { return arrow::Status::OK(); } - arrow::Status Visit(const arrow::FloatType &type) final { + arrow::Status Visit(const arrow::FloatType &) final { inner_.template operator()(); return arrow::Status::OK(); } - arrow::Status Visit(const arrow::DoubleType &type) final { + arrow::Status Visit(const arrow::DoubleType &) final { inner_.template operator()(); return arrow::Status::OK(); } - arrow::Status Visit(const arrow::StringType &type) final { + arrow::Status Visit(const arrow::BooleanType &) final { + inner_.template operator()(); + return arrow::Status::OK(); + } + + arrow::Status Visit(const arrow::StringType &) final { inner_.template operator()(); return arrow::Status::OK(); } + arrow::Status Visit(const arrow::TimestampType &) final { + inner_.template operator()(); + return arrow::Status::OK(); + } + Inner &inner() { return inner_; } const Inner &inner() const { return inner_; } @@ -96,10 +107,45 @@ class ArrowArrayTypeVisitor : public arrow::ArrayVisitor { return arrow::Status::OK(); } + arrow::Status Visit(const arrow::BooleanArray &) final { + inner_.template operator()(); + return arrow::Status::OK(); + } + + arrow::Status Visit(const arrow::TimestampArray &) final { + inner_.template operator()(); + return arrow::Status::OK(); + } + Inner &inner() { return inner_; } const Inner &inner() const { return inner_; } private: Inner inner_; }; + +/** + * Returns true iff type T is one of the numeric types. + */ +template +constexpr bool isNumericType() { + static_assert( + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v, + "T is not one of the supported element types for Deephaven columns"); + + return std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v; +} } // namespace deephaven::client::arrowutil diff --git a/cpp-client/deephaven/client/include/private/deephaven/client/immerutil/abstract_flex_vector.h b/cpp-client/deephaven/client/include/private/deephaven/client/immerutil/abstract_flex_vector.h index 1c36f17b3b0..112c2306bdf 100644 --- a/cpp-client/deephaven/client/include/private/deephaven/client/immerutil/abstract_flex_vector.h +++ b/cpp-client/deephaven/client/include/private/deephaven/client/immerutil/abstract_flex_vector.h @@ -7,6 +7,7 @@ #include #include #include +#include "deephaven/client/arrowutil/arrow_value_converter.h" #include "deephaven/client/column/column_source.h" #include "deephaven/client/immerutil/immer_column_source.h" #include "deephaven/client/utility/utility.h" @@ -46,38 +47,53 @@ struct CorrespondingArrowArrayType { typedef arrow::DoubleArray type_t; }; +template<> +struct CorrespondingArrowArrayType { + typedef arrow::BooleanArray type_t; +}; + template<> struct CorrespondingArrowArrayType { typedef arrow::StringArray type_t; }; +template<> +struct CorrespondingArrowArrayType { + typedef arrow::TimestampArray type_t; +}; + struct FlexVectorAppender { template - static void append(const ARROW_SRC &src, immer::flex_vector *dest) { - auto transient = dest->transient(); - for (auto element : src) { - if (!element.has_value()) { - throw std::runtime_error("TODO(kosak): we are not dealing with null values yet"); - } - transient.push_back(*element); - } - *dest = transient.persistent(); - } + static void append(const ARROW_SRC &src, immer::flex_vector *destData, + immer::flex_vector *optionalDestNulls) { - static void append(const arrow::StringArray &src, immer::flex_vector *dest) { - auto transient = dest->transient(); - for (auto element : src) { - if (!element.has_value()) { - throw std::runtime_error("TODO(kosak): we are not dealing with null values yet"); + auto transientData = destData->transient(); + immer::flex_vector_transient transientNulls; + if (optionalDestNulls != nullptr) { + transientNulls = optionalDestNulls->transient(); + } + for (auto optValue : src) { + bool isNull = !optValue.has_value(); + T value = T(); + if (isNull) { + if constexpr(deephaven::client::arrowutil::isNumericType()) { + value = deephaven::client::DeephavenConstantsForType::NULL_VALUE; + } + } else { + deephaven::client::arrowutil::ArrowValueConverter::convert(*optValue, &value); } - transient.push_back(std::string(*element)); + transientData.push_back(std::move(value)); + if (optionalDestNulls != nullptr) { + transientNulls.push_back(isNull); + } + } + *destData = transientData.persistent(); + if (optionalDestNulls != nullptr) { + *optionalDestNulls = transientNulls.persistent(); } - *dest = transient.persistent(); } }; } // namespace internal -template -class AbstractFlexVector; /** * This class allows us to manipulate an immer::flex_vector without needing to know what type @@ -87,9 +103,6 @@ class AbstractFlexVectorBase { protected: typedef deephaven::client::column::ColumnSource ColumnSource; public: - template - static std::unique_ptr create(immer::flex_vector vec); - virtual ~AbstractFlexVectorBase(); virtual std::unique_ptr take(size_t n) = 0; @@ -101,12 +114,16 @@ class AbstractFlexVectorBase { }; template -class AbstractFlexVector final : public AbstractFlexVectorBase { +class NumericAbstractFlexVector final : public AbstractFlexVectorBase { public: - explicit AbstractFlexVector(immer::flex_vector vec) : vec_(std::move(vec)) {} + NumericAbstractFlexVector() = default; + + explicit NumericAbstractFlexVector(immer::flex_vector vec) : vec_(std::move(vec)) {} + + ~NumericAbstractFlexVector() final = default; std::unique_ptr take(size_t n) final { - return create(vec_.take(n)); + return std::make_unique(vec_.take(n)); } void inPlaceDrop(size_t n) final { @@ -115,21 +132,21 @@ class AbstractFlexVector final : public AbstractFlexVectorBase { } void inPlaceAppend(std::unique_ptr other) final { - auto *otherVec = deephaven::client::utility::verboseCast(other.get(), - DEEPHAVEN_PRETTY_FUNCTION); + auto *otherVec = deephaven::client::utility::verboseCast( + DEEPHAVEN_EXPR_MSG(other.get())); auto temp = std::move(vec_) + std::move(otherVec->vec_); vec_ = std::move(temp); } void inPlaceAppendArrow(const arrow::Array &data) final { typedef typename internal::CorrespondingArrowArrayType::type_t arrowArrayType_t; - auto *typedArrow = deephaven::client::utility::verboseCast(&data, - DEEPHAVEN_PRETTY_FUNCTION); - internal::FlexVectorAppender::append(*typedArrow, &vec_); + auto *typedArrow = deephaven::client::utility::verboseCast( + DEEPHAVEN_EXPR_MSG(&data)); + internal::FlexVectorAppender::append(*typedArrow, &vec_, nullptr); } std::shared_ptr makeColumnSource() const final { - return deephaven::client::immerutil::ImmerColumnSource::create(vec_); + return deephaven::client::immerutil::NumericImmerColumnSource::create(vec_); } private: @@ -137,7 +154,50 @@ class AbstractFlexVector final : public AbstractFlexVectorBase { }; template -std::unique_ptr AbstractFlexVectorBase::create(immer::flex_vector vec) { - return std::make_unique>(std::move(vec)); -} +class GenericAbstractFlexVector final : public AbstractFlexVectorBase { +public: + GenericAbstractFlexVector() = default; + + GenericAbstractFlexVector(immer::flex_vector data, immer::flex_vector nulls) : + data_(std::move(data)), nulls_(std::move(nulls)) {} + + ~GenericAbstractFlexVector() final = default; + + std::unique_ptr take(size_t n) final { + return std::make_unique(data_.take(n), nulls_.take(n)); + } + + void inPlaceDrop(size_t n) final { + auto tempData = std::move(data_).drop(n); + data_ = std::move(tempData); + + auto tempNulls = std::move(nulls_).drop(n); + nulls_ = std::move(tempNulls); + } + + void inPlaceAppend(std::unique_ptr other) final { + auto *otherVec = deephaven::client::utility::verboseCast( + DEEPHAVEN_EXPR_MSG(other.get())); + auto tempData = std::move(data_) + std::move(otherVec->data_); + data_ = std::move(tempData); + + auto tempNulls = std::move(nulls_) + std::move(otherVec->nulls_); + nulls_ = std::move(tempNulls); + } + + void inPlaceAppendArrow(const arrow::Array &data) final { + typedef typename internal::CorrespondingArrowArrayType::type_t arrowArrayType_t; + auto *typedArrow = deephaven::client::utility::verboseCast( + DEEPHAVEN_EXPR_MSG(&data)); + internal::FlexVectorAppender::append(*typedArrow, &data_, &nulls_); + } + + std::shared_ptr makeColumnSource() const final { + return deephaven::client::immerutil::GenericImmerColumnSource::create(data_, nulls_); + } + +private: + immer::flex_vector data_; + immer::flex_vector nulls_; +}; } // namespace deephaven::client::immerutil diff --git a/cpp-client/deephaven/client/include/private/deephaven/client/immerutil/immer_column_source.h b/cpp-client/deephaven/client/include/private/deephaven/client/immerutil/immer_column_source.h index 74881645108..fc68a6c997a 100644 --- a/cpp-client/deephaven/client/include/private/deephaven/client/immerutil/immer_column_source.h +++ b/cpp-client/deephaven/client/include/private/deephaven/client/immerutil/immer_column_source.h @@ -4,78 +4,137 @@ #pragma once #include #include +#include "deephaven/client/arrowutil/arrow_visitors.h" #include "deephaven/client/chunk/chunk.h" #include "deephaven/client/column/column_source.h" +#include "deephaven/client/types.h" #include "deephaven/client/utility/utility.h" namespace deephaven::client::immerutil { namespace internal { -struct ColumnSourceImpls { +struct ImmerColumnSourceImpls { + typedef deephaven::client::chunk::BooleanChunk BooleanChunk; typedef deephaven::client::chunk::Chunk Chunk; typedef deephaven::client::chunk::UInt64Chunk UInt64Chunk; typedef deephaven::client::container::RowSequence RowSequence; + /** + * This helper function has two dimensions of optionality: the first (controlled by whether the + * template is numeric) indicates whether (if it is numeric) "null-ness" comes from the inherent + * Deephaven notion of null-ness (the special numeric constants), or (if it is not numeric), + * it comes from a separate vector of null flags. The second dimension of optionality is + * controlled by 'optionalDestNullFlags', which indicates whether the caller cares about nullness. + * If this pointer is not null, then it points to a BooleanChunk which can hold all the null + * flags. On the other hand if this pointer is null, then the caller doesn't care about null flags + * and we don't have to do any special work to determine nullness. + */ + template - static void fillChunk(const immer::flex_vector &src, const RowSequence &rows, Chunk *dest) { + static void fillChunk(const immer::flex_vector &srcData, + const immer::flex_vector *srcNullFlags, + const RowSequence &rows, Chunk *destData, BooleanChunk *optionalDestNullFlags) { using deephaven::client::chunk::TypeToChunk; - using deephaven::client::utility::assertLessEq; + using deephaven::client::utility::trueOrThrow; using deephaven::client::utility::verboseCast; typedef typename TypeToChunk::type_t chunkType_t; - auto *typedDest = verboseCast(dest, DEEPHAVEN_PRETTY_FUNCTION); - - assertLessEq(rows.size(), typedDest->size(), DEEPHAVEN_PRETTY_FUNCTION, "rows.size()", - "typedDest->size()"); - auto *destp = typedDest->data(); + auto *typedDest = verboseCast(DEEPHAVEN_EXPR_MSG(destData)); + + constexpr bool typeIsNumeric = deephaven::client::arrowutil::isNumericType(); + + trueOrThrow(DEEPHAVEN_EXPR_MSG(rows.size() <= typedDest->size())); + trueOrThrow(DEEPHAVEN_EXPR_MSG(optionalDestNullFlags == nullptr || + rows.size() <= optionalDestNullFlags->size())); + if (!typeIsNumeric) { + trueOrThrow(DEEPHAVEN_EXPR_MSG(srcNullFlags != nullptr)); + } else { + // avoid CLion warning about unused variable. + (void)srcNullFlags; + } + auto *destDatap = typedDest->data(); + auto *destNullp = optionalDestNullFlags != nullptr ? optionalDestNullFlags->data() : nullptr; - auto copyDataInner = [&destp](const T *dataBegin, const T *dataEnd) { + auto copyDataInner = [&destDatap, &destNullp](const T *dataBegin, const T *dataEnd) { for (const T *current = dataBegin; current != dataEnd; ++current) { - *destp++ = *current; + auto value = *current; + *destDatap++ = value; + if constexpr(typeIsNumeric) { + if (destNullp != nullptr) { + *destNullp++ = value == deephaven::client::DeephavenConstantsForType::NULL_VALUE; + } + } + } + }; + + auto copyNullsInner = [&destNullp](const bool *nullBegin, const bool *nullEnd) { + for (const bool *current = nullBegin; current != nullEnd; ++current) { + *destNullp++ = *current; } }; - auto copyDataOuter = [&src, ©DataInner](uint64_t srcBegin, uint64_t srcEnd) { - auto srcBeginp = src.begin() + srcBegin; - auto srcEndp = src.begin() + srcEnd; + auto copyOuter = [&srcData, srcNullFlags, destNullp, ©DataInner, + ©NullsInner](uint64_t srcBegin, uint64_t srcEnd) { + auto srcBeginp = srcData.begin() + srcBegin; + auto srcEndp = srcData.begin() + srcEnd; immer::for_each_chunk(srcBeginp, srcEndp, copyDataInner); + + if constexpr(!typeIsNumeric) { + if (destNullp != nullptr) { + auto nullsBeginp = srcNullFlags->begin() + srcBegin; + auto nullsEndp = srcNullFlags->begin() + srcEnd; + immer::for_each_chunk(nullsBeginp, nullsEndp, copyNullsInner); + } + } }; - rows.forEachChunk(copyDataOuter); + rows.forEachChunk(copyOuter); } template - static void fillChunkUnordered(const immer::flex_vector &src, const UInt64Chunk &rowKeys, - Chunk *dest) { + static void fillChunkUnordered(const immer::flex_vector &srcData, + const immer::flex_vector *srcNullFlags, + const UInt64Chunk &rowKeys, Chunk *destData, BooleanChunk *optionalDestNullFlags) { using deephaven::client::chunk::TypeToChunk; - using deephaven::client::utility::assertLessEq; + using deephaven::client::utility::trueOrThrow; using deephaven::client::utility::verboseCast; typedef typename TypeToChunk::type_t chunkType_t; - auto *typedDest = verboseCast(dest, DEEPHAVEN_PRETTY_FUNCTION); - assertLessEq(rowKeys.size(), typedDest->size(), DEEPHAVEN_PRETTY_FUNCTION, "rowKeys.size()", - "typedDest->size()"); + constexpr bool typeIsNumeric = deephaven::client::arrowutil::isNumericType(); + + auto *typedDest = verboseCast(DEEPHAVEN_EXPR_MSG(destData)); + trueOrThrow(DEEPHAVEN_EXPR_MSG(rowKeys.size() <= typedDest->size())); + trueOrThrow(DEEPHAVEN_EXPR_MSG(optionalDestNullFlags == nullptr || + rowKeys.size() <= optionalDestNullFlags->size())); + if (!typeIsNumeric) { + trueOrThrow(DEEPHAVEN_EXPR_MSG(srcNullFlags != nullptr)); + } auto *destp = typedDest->data(); + auto *destNullp = optionalDestNullFlags != nullptr ? optionalDestNullFlags->data() : nullptr; - // Note: Uses random access with Immer, which is rather more expensive than iterating + // Note: Uses random access with Immer, which is significantly more expensive than iterating // over contiguous Immer ranges. for (auto key : rowKeys) { - *destp++ = src[key]; + auto value = srcData[key]; + *destp++ = value; + if (destNullp != nullptr) { + if constexpr(typeIsNumeric) { + *destNullp++ = value == deephaven::client::DeephavenConstantsForType::NULL_VALUE; + } else { + *destNullp++ = (*srcNullFlags)[key]; + } + } } } }; } // namespace internal class ImmerColumnSource : public virtual deephaven::client::column::ColumnSource { -public: - template - static std::shared_ptr create(immer::flex_vector vec); - - static std::shared_ptr create(immer::flex_vector vec); }; template -class ImmerNumericColumnSource final : public ImmerColumnSource, +class NumericImmerColumnSource final : public ImmerColumnSource, public deephaven::client::column::NumericColumnSource, - std::enable_shared_from_this> { + std::enable_shared_from_this> { + struct Private {}; typedef deephaven::client::chunk::Chunk Chunk; typedef deephaven::client::chunk::UInt64Chunk UInt64Chunk; @@ -83,15 +142,23 @@ class ImmerNumericColumnSource final : public ImmerColumnSource, typedef deephaven::client::container::RowSequence RowSequence; public: - explicit ImmerNumericColumnSource(immer::flex_vector data) : data_(std::move(data)) {} - ~ImmerNumericColumnSource() final = default; + static std::shared_ptr create(immer::flex_vector data) { + return std::make_shared(Private(), std::move(data)); + } + + explicit NumericImmerColumnSource(Private, immer::flex_vector data) : data_(std::move(data)) {} + + ~NumericImmerColumnSource() final = default; - void fillChunk(const RowSequence &rows, Chunk *dest) const final { - internal::ColumnSourceImpls::fillChunk(data_, rows, dest); + void fillChunk(const RowSequence &rows, Chunk *destData, BooleanChunk *optionalDestNullFlags) const final { + internal::ImmerColumnSourceImpls::fillChunk(data_, nullptr, rows, destData, + optionalDestNullFlags); } - void fillChunkUnordered(const UInt64Chunk &rowKeys, Chunk *dest) const final { - internal::ColumnSourceImpls::fillChunkUnordered(data_, rowKeys, dest); + void fillChunkUnordered(const UInt64Chunk &rowKeys, Chunk *destData, + BooleanChunk *optionalDestNullFlags) const final { + internal::ImmerColumnSourceImpls::fillChunkUnordered(data_, nullptr, rowKeys, destData, + optionalDestNullFlags); } void acceptVisitor(ColumnSourceVisitor *visitor) const final { @@ -106,37 +173,43 @@ class ImmerNumericColumnSource final : public ImmerColumnSource, immer::flex_vector data_; }; -class ImmerStringColumnSource final : public ImmerColumnSource, - public deephaven::client::column::StringColumnSource, - std::enable_shared_from_this { +template +class GenericImmerColumnSource final : public ImmerColumnSource, + public deephaven::client::column::GenericColumnSource, + std::enable_shared_from_this> { + struct Private {}; typedef deephaven::client::column::ColumnSourceVisitor ColumnSourceVisitor; public: - explicit ImmerStringColumnSource(immer::flex_vector data) : data_(std::move(data)) {} - ~ImmerStringColumnSource() final = default; + static std::shared_ptr create(immer::flex_vector data, + immer::flex_vector nullFlags) { + return std::make_shared(Private(), std::move(data), std::move(nullFlags)); + } + + GenericImmerColumnSource(Private, immer::flex_vector &&data, immer::flex_vector &&nullFlags) : + data_(std::move(data)), nullFlags_(std::move(nullFlags)) {} + ~GenericImmerColumnSource() final = default; - void fillChunk(const RowSequence &rows, Chunk *dest) const final { - internal::ColumnSourceImpls::fillChunk(data_, rows, dest); + void fillChunk(const RowSequence &rows, Chunk *dest, BooleanChunk *optionalDestNullFlags) const final { + internal::ImmerColumnSourceImpls::fillChunk(data_, &nullFlags_, rows, dest, + optionalDestNullFlags); } - void fillChunkUnordered(const UInt64Chunk &rowKeys, Chunk *dest) const final { - internal::ColumnSourceImpls::fillChunkUnordered(data_, rowKeys, dest); + void fillChunkUnordered(const UInt64Chunk &rowKeys, Chunk *dest, + BooleanChunk *optionalDestNullFlags) const final { + internal::ImmerColumnSourceImpls::fillChunkUnordered(data_, &nullFlags_, rowKeys, dest, + optionalDestNullFlags); } void acceptVisitor(ColumnSourceVisitor *visitor) const final { visitor->visit(*this); } + std::any backdoor() const final { + return &data_; + } + private: - immer::flex_vector data_; + immer::flex_vector data_; + immer::flex_vector nullFlags_; }; - -template -std::shared_ptr ImmerColumnSource::create(immer::flex_vector vec) { - return std::make_shared>(std::move(vec)); -} - -inline std::shared_ptr ImmerColumnSource::create( - immer::flex_vector vec) { - return std::make_shared(std::move(vec)); -} } // namespace deephaven::client::immerutil diff --git a/cpp-client/deephaven/client/include/private/deephaven/client/impl/table_handle_impl.h b/cpp-client/deephaven/client/include/private/deephaven/client/impl/table_handle_impl.h index f4c6a208f38..d567548337d 100644 --- a/cpp-client/deephaven/client/include/private/deephaven/client/impl/table_handle_impl.h +++ b/cpp-client/deephaven/client/include/private/deephaven/client/impl/table_handle_impl.h @@ -176,8 +176,7 @@ class TableHandleImpl { void bindToVariableAsync(std::string variable, std::shared_ptr> callback); - std::shared_ptr subscribe(std::shared_ptr callback, - bool wantImmer); + std::shared_ptr subscribe(std::shared_ptr callback); void unsubscribe(std::shared_ptr handle); // For debugging diff --git a/cpp-client/deephaven/client/include/private/deephaven/client/subscription/batch_parser.h b/cpp-client/deephaven/client/include/private/deephaven/client/subscription/batch_parser.h index 9cbff87760f..dfc5ad373ca 100644 --- a/cpp-client/deephaven/client/include/private/deephaven/client/subscription/batch_parser.h +++ b/cpp-client/deephaven/client/include/private/deephaven/client/subscription/batch_parser.h @@ -15,7 +15,7 @@ class BatchParser { BatchParser() = delete; static void parseBatches( - const ColumnDefinitions &colDefs, + size_t expectedNumCols, size_t numBatches, bool allowInconsistentColumnSizes, arrow::flight::FlightStreamReader *fsr, diff --git a/cpp-client/deephaven/client/include/private/deephaven/client/subscription/classic_table_state.h b/cpp-client/deephaven/client/include/private/deephaven/client/subscription/classic_table_state.h index d3c42f3f583..1a9ff4241d2 100644 --- a/cpp-client/deephaven/client/include/private/deephaven/client/subscription/classic_table_state.h +++ b/cpp-client/deephaven/client/include/private/deephaven/client/subscription/classic_table_state.h @@ -16,6 +16,7 @@ class ClassicTableState final { typedef deephaven::client::column::ColumnSource ColumnSource; typedef deephaven::client::column::MutableColumnSource MutableColumnSource; typedef deephaven::client::container::RowSequence RowSequence; + typedef deephaven::client::table::Schema Schema; typedef deephaven::client::table::Table Table; typedef deephaven::client::utility::ColumnDefinitions ColumnDefinitions; @@ -43,6 +44,7 @@ class ClassicTableState final { private: UInt64Chunk modifyKeysHelper(const RowSequence &rowsToModifyKeySpace); + std::shared_ptr schema_; std::vector> columns_; std::shared_ptr> redirection_; /** diff --git a/cpp-client/deephaven/client/include/private/deephaven/client/subscription/immer_table_state.h b/cpp-client/deephaven/client/include/private/deephaven/client/subscription/immer_table_state.h index 83d44429a44..7c1c86aa70e 100644 --- a/cpp-client/deephaven/client/include/private/deephaven/client/subscription/immer_table_state.h +++ b/cpp-client/deephaven/client/include/private/deephaven/client/subscription/immer_table_state.h @@ -16,11 +16,12 @@ class ImmerTableState final { typedef deephaven::client::column::ColumnSource ColumnSource; typedef deephaven::client::container::RowSequence RowSequence; typedef deephaven::client::immerutil::AbstractFlexVectorBase AbstractFlexVectorBase; + typedef deephaven::client::table::Schema Schema; typedef deephaven::client::table::Table Table; typedef deephaven::client::utility::ColumnDefinitions ColumnDefinitions; public: - explicit ImmerTableState(const ColumnDefinitions &colDefs); + explicit ImmerTableState(std::shared_ptr colDefs); ~ImmerTableState(); std::shared_ptr addKeys(const RowSequence &rowsToAddKeySpace); @@ -44,6 +45,8 @@ class ImmerTableState final { std::unique_ptr modifiedData, const RowSequence &rowsToModifyKeySpace); + std::shared_ptr colDefs_; + std::shared_ptr schema_; std::vector> flexVectors_; // Keeps track of keyspace -> index space mapping SpaceMapper spaceMapper_; diff --git a/cpp-client/deephaven/client/include/private/deephaven/client/subscription/subscribe_thread.h b/cpp-client/deephaven/client/include/private/deephaven/client/subscription/subscribe_thread.h index 04999199aba..004e20cff0c 100644 --- a/cpp-client/deephaven/client/include/private/deephaven/client/subscription/subscribe_thread.h +++ b/cpp-client/deephaven/client/include/private/deephaven/client/subscription/subscribe_thread.h @@ -16,6 +16,5 @@ std::shared_ptr startSubscribeThread( deephaven::client::utility::Executor *flightExecutor, std::shared_ptr columnDefinitions, const io::deephaven::proto::backplane::grpc::Ticket &ticket, - std::shared_ptr callback, - bool wantImmer); + std::shared_ptr callback); } // namespace deephaven::client::subscription diff --git a/cpp-client/deephaven/client/include/private/deephaven/client/subscription/update_processor.h b/cpp-client/deephaven/client/include/private/deephaven/client/subscription/update_processor.h index 959b719bd33..7bed257455a 100644 --- a/cpp-client/deephaven/client/include/private/deephaven/client/subscription/update_processor.h +++ b/cpp-client/deephaven/client/include/private/deephaven/client/subscription/update_processor.h @@ -3,6 +3,7 @@ */ #pragma once +#include #include #include #include "deephaven/client/ticking.h" @@ -17,8 +18,7 @@ class UpdateProcessor final { static std::shared_ptr startThread( std::unique_ptr fsr, std::shared_ptr colDefs, - std::shared_ptr callback, - bool wantImmer); + std::shared_ptr callback); UpdateProcessor(std::unique_ptr fsr, std::shared_ptr colDefs, std::shared_ptr callback); @@ -27,13 +27,13 @@ class UpdateProcessor final { void cancel(); private: - static void runForever(const std::shared_ptr &self, bool wantImmer); - void classicRunForeverHelper(); - void immerRunForeverHelper(); + static void runForever(const std::shared_ptr &self); + void runForeverHelper(); public: std::unique_ptr fsr_; std::shared_ptr colDefs_; std::shared_ptr callback_; + std::atomic cancelled_; }; } // namespace deephaven::client::subscription diff --git a/cpp-client/deephaven/client/include/private/deephaven/client/utility/misc.h b/cpp-client/deephaven/client/include/private/deephaven/client/utility/misc.h index ceb16de63df..a0b94dac782 100644 --- a/cpp-client/deephaven/client/include/private/deephaven/client/utility/misc.h +++ b/cpp-client/deephaven/client/include/private/deephaven/client/utility/misc.h @@ -22,6 +22,8 @@ class ColumnDefinitions { const map_t &map() const { return map_; } + size_t size() const { return vec_.size(); } + private: vec_t vec_; map_t map_; diff --git a/cpp-client/deephaven/client/include/public/deephaven/client/chunk/chunk.h b/cpp-client/deephaven/client/include/public/deephaven/client/chunk/chunk.h index 479e162bc30..1f1d6d61642 100644 --- a/cpp-client/deephaven/client/include/public/deephaven/client/chunk/chunk.h +++ b/cpp-client/deephaven/client/include/public/deephaven/client/chunk/chunk.h @@ -7,6 +7,7 @@ #include #include #include +#include "deephaven/client/types.h" #include "deephaven/client/utility/utility.h" namespace deephaven::client::chunk { @@ -63,6 +64,7 @@ class GenericChunk final : public Chunk { std::shared_ptr data_; }; +typedef GenericChunk BooleanChunk; typedef GenericChunk Int8Chunk; typedef GenericChunk Int16Chunk; typedef GenericChunk Int32Chunk; @@ -71,13 +73,14 @@ typedef GenericChunk UInt64Chunk; typedef GenericChunk FloatChunk; typedef GenericChunk DoubleChunk; typedef GenericChunk StringChunk; +typedef GenericChunk DateTimeChunk; /** * Typesafe union of all the Chunk types. */ class AnyChunk { - typedef std::variant variant_t; + typedef std::variant variant_t; public: template @@ -145,7 +148,9 @@ class ChunkVisitor { virtual void visit(const UInt64Chunk &) const = 0; virtual void visit(const FloatChunk &) const = 0; virtual void visit(const DoubleChunk &) const = 0; + virtual void visit(const BooleanChunk &) const = 0; virtual void visit(const StringChunk &) const = 0; + virtual void visit(const DateTimeChunk &) const = 0; }; template @@ -186,8 +191,18 @@ struct TypeToChunk { typedef deephaven::client::chunk::DoubleChunk type_t; }; +template<> +struct TypeToChunk { + typedef deephaven::client::chunk::BooleanChunk type_t; +}; + template<> struct TypeToChunk { typedef deephaven::client::chunk::StringChunk type_t; }; + +template<> +struct TypeToChunk { + typedef deephaven::client::chunk::DateTimeChunk type_t; +}; } // namespace deephaven::client::chunk diff --git a/cpp-client/deephaven/client/include/public/deephaven/client/chunk/chunk_filler.h b/cpp-client/deephaven/client/include/public/deephaven/client/chunk/chunk_filler.h index 9ea7f499c31..c47902b315b 100644 --- a/cpp-client/deephaven/client/include/public/deephaven/client/chunk/chunk_filler.h +++ b/cpp-client/deephaven/client/include/public/deephaven/client/chunk/chunk_filler.h @@ -13,6 +13,7 @@ namespace deephaven::client::chunk { class ChunkFiller { typedef deephaven::client::container::RowSequence RowSequence; public: - static void fillChunk(const arrow::Array &src, const RowSequence &keys, Chunk *dest); + static void fillChunk(const arrow::Array &src, const RowSequence &keys, Chunk *destData, + BooleanChunk *optionalDestNullFlags); }; } // namespace deephaven::client::chunk diff --git a/cpp-client/deephaven/client/include/public/deephaven/client/client.h b/cpp-client/deephaven/client/include/public/deephaven/client/client.h index a88d36231c9..4fa7686becc 100644 --- a/cpp-client/deephaven/client/include/public/deephaven/client/client.h +++ b/cpp-client/deephaven/client/include/public/deephaven/client/client.h @@ -1081,12 +1081,9 @@ class TableHandle { std::shared_ptr getFlightStreamReader() const; /** - * Early unstable interface to subscribe/unsubscribe to ticking tables. This interface supports - * append-only tables and will call back with an error if the table changes in a way that is not - * append-only. + * Subscribe to a ticking table. */ - std::shared_ptr subscribe(std::shared_ptr callback, - bool wantImmer); + std::shared_ptr subscribe(std::shared_ptr callback); /** * Unsubscribe from the table. */ diff --git a/cpp-client/deephaven/client/include/public/deephaven/client/column/array_column_source.h b/cpp-client/deephaven/client/include/public/deephaven/client/column/array_column_source.h new file mode 100644 index 00000000000..eeb8d21004a --- /dev/null +++ b/cpp-client/deephaven/client/include/public/deephaven/client/column/array_column_source.h @@ -0,0 +1,325 @@ +/** + * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending + */ + +#pragma once + +#include "deephaven/client/column/column_source.h" +#include "deephaven/client/types.h" + +namespace deephaven::client::column { +namespace internal { +// A central place to put the implementations for the similar-but-not-identical +// fill{,From}Chunk{,Unordered} implementations for the various column source types. + +struct ColumnSourceImpls { + typedef deephaven::client::chunk::BooleanChunk BooleanChunk; + typedef deephaven::client::chunk::Chunk Chunk; + typedef deephaven::client::container::RowSequence RowSequence; + typedef deephaven::client::chunk::UInt64Chunk UInt64Chunk; + + template + static void fillChunk(const RowSequence &rows, + Chunk *dest, BooleanChunk *optionalNullFlags, const BACKING_STORE &backingStore) { + using deephaven::client::utility::trueOrThrow; + using deephaven::client::utility::verboseCast; + + auto *typedDest = verboseCast(DEEPHAVEN_EXPR_MSG(dest)); + trueOrThrow(DEEPHAVEN_EXPR_MSG(rows.size() <= typedDest->size())); + auto *destData = typedDest->data(); + auto *destNull = optionalNullFlags != nullptr ? optionalNullFlags->data() : nullptr; + size_t destIndex = 0; + auto applyChunk = [destData, destNull, &backingStore, &destIndex] + (uint64_t begin, uint64_t end) { + for (auto srcIndex = begin; srcIndex != end; ++srcIndex) { + auto [value, isNull] = backingStore.get(srcIndex); + destData[destIndex] = std::move(value); + if (destNull != nullptr) { + destNull[destIndex] = isNull; + } + ++destIndex; + } + }; + rows.forEachChunk(applyChunk); + } + + template + static void fillChunkUnordered( + const UInt64Chunk &rowKeys, Chunk *dest, BooleanChunk *optionalNullFlags, + const BACKING_STORE &backingStore) { + using deephaven::client::chunk::TypeToChunk; + using deephaven::client::utility::trueOrThrow; + using deephaven::client::utility::verboseCast; + + auto *typedDest = verboseCast(DEEPHAVEN_EXPR_MSG(dest)); + trueOrThrow(DEEPHAVEN_EXPR_MSG(rowKeys.size() <= typedDest->size())); + const uint64_t *keys = rowKeys.data(); + auto *destData = typedDest->data(); + auto *destNull = optionalNullFlags != nullptr ? optionalNullFlags->data() : nullptr; + + for (size_t destIndex = 0; destIndex < rowKeys.size(); ++destIndex) { + auto srcIndex = keys[destIndex]; + auto [value, isNull] = backingStore.get(srcIndex); + destData[destIndex] = std::move(value); + if (destNull != nullptr) { + destNull[destIndex] = isNull; + } + } + } + + template + static void fillFromChunk(const Chunk &src, const BooleanChunk *optionalSrcNullFlags, + const RowSequence &rows, BACKING_STORE *backingStore) { + using deephaven::client::chunk::TypeToChunk; + using deephaven::client::utility::trueOrThrow; + using deephaven::client::utility::verboseCast; + + const auto *typedSrc = verboseCast(DEEPHAVEN_EXPR_MSG(&src)); + trueOrThrow(DEEPHAVEN_EXPR_MSG(rows.size() <= typedSrc->size())); + + const auto *srcData = typedSrc->data(); + const auto *nullData = optionalSrcNullFlags != nullptr ? optionalSrcNullFlags->data() : nullptr; + size_t srcIndex = 0; + auto applyChunk = [srcData, &srcIndex, nullData, backingStore](uint64_t begin, uint64_t end) { + backingStore->ensureCapacity(end); + for (auto destIndex = begin; destIndex != end; ++destIndex) { + auto value = srcData[srcIndex]; + auto forceNull = nullData != nullptr && nullData[srcIndex]; + backingStore->set(destIndex, value, forceNull); + ++srcIndex; + } + }; + rows.forEachChunk(applyChunk); + } + + template + static void fillFromChunkUnordered(const Chunk &src, const BooleanChunk *optionalSrcNullFlags, + const UInt64Chunk &rowKeys, BACKING_STORE *backingStore) { + using deephaven::client::chunk::TypeToChunk; + using deephaven::client::utility::trueOrThrow; + using deephaven::client::utility::verboseCast; + + const auto *typedSrc = verboseCast(DEEPHAVEN_EXPR_MSG(&src)); + trueOrThrow(DEEPHAVEN_EXPR_MSG(rowKeys.size() <= typedSrc->size())); + trueOrThrow(DEEPHAVEN_EXPR_MSG(optionalSrcNullFlags == nullptr || + rowKeys.size() <= optionalSrcNullFlags->size())); + + const auto *keyData = rowKeys.data(); + const auto *srcData = typedSrc->data(); + const auto *nullData = optionalSrcNullFlags != nullptr ? optionalSrcNullFlags->data() : nullptr; + for (size_t srcIndex = 0; srcIndex < typedSrc->size(); ++srcIndex) { + auto destIndex = keyData[srcIndex]; + backingStore->ensureCapacity(destIndex + 1); + auto value = srcData[srcIndex]; + auto forceNull = nullData != nullptr && nullData[srcIndex]; + backingStore->set(destIndex, value, forceNull); + } + } + +}; + +class BackingStoreBase { +protected: + void assertIndexValid(size_t index) const; + + template + void growIfNeeded(size_t requestedCapacity, std::unique_ptr *data, + std::unique_ptr *optionalNullFlags) { + if (requestedCapacity <= capacity_) { + return; + } + auto oldCapacity = capacity_; + // sad + if (capacity_ == 0) { + capacity_ = 1; + } + while (capacity_ < requestedCapacity) { + capacity_ *= 2; + } + + auto newData = std::make_unique(capacity_); + auto newNullFlags = + optionalNullFlags != nullptr ? std::make_unique(capacity_) : nullptr; + std::copy( + std::make_move_iterator(data->get()), + std::make_move_iterator(data->get() + oldCapacity), + newData.get()); + *data = std::move(newData); + if (optionalNullFlags != nullptr) { + std::copy(optionalNullFlags->get(), + optionalNullFlags->get() + oldCapacity, + newNullFlags.get()); + *optionalNullFlags = std::move(newNullFlags); + } + } + + size_t capacity_ = 0; +}; + +/** + * This is the backing store used for the numeric types, which have the property that "null" is + * represented by a special Deephaven constant. + */ +template +class NumericBackingStore : public BackingStoreBase { +public: + NumericBackingStore() { + data_ = std::make_unique(0); + } + + std::tuple get(size_t index) const { + assertIndexValid(index); + auto value = data_[index]; + auto isNull = value == deephaven::client::DeephavenConstantsForType::NULL_VALUE; + return std::make_tuple(value, isNull); + } + + void set(size_t index, T value, bool forceNull) { + assertIndexValid(index); + if (forceNull) { + value = deephaven::client::DeephavenConstantsForType::NULL_VALUE; + } + data_[index] = value; + } + + void ensureCapacity(size_t requestedCapacity) { + growIfNeeded(requestedCapacity, &data_, nullptr); + } + +private: + std::unique_ptr data_; +}; + +/** + * This is the backing store used for other types like std::string, bool, and DateTime, which track + * the "null" flag explicitly. + */ +template +class GenericBackingStore : public BackingStoreBase { +public: + GenericBackingStore() { + data_ = std::make_unique(0); + } + + std::tuple get(size_t index) const { + assertIndexValid(index); + auto value = data_[index]; + auto isNull = isNull_[index]; + return std::make_tuple(value, isNull); + } + + void set(size_t index, T value, bool forceNull) { + assertIndexValid(index); + data_[index] = std::move(value); + isNull_[index] = forceNull; + } + + void ensureCapacity(size_t requestedCapacity) { + growIfNeeded(requestedCapacity, &data_, &isNull_); + } + +private: + std::unique_ptr data_; + std::unique_ptr isNull_; +}; +} // namespace internal + +template +class NumericArrayColumnSource final : public MutableNumericColumnSource, + std::enable_shared_from_this> { + struct Private { + }; + typedef deephaven::client::chunk::BooleanChunk BooleanChunk; + typedef deephaven::client::chunk::Chunk Chunk; + typedef deephaven::client::chunk::UInt64Chunk UInt64Chunk; + typedef deephaven::client::container::RowSequence RowSequence; + +public: + static std::shared_ptr create() { + return std::make_shared>(Private()); + } + + explicit NumericArrayColumnSource(Private) {} + + ~NumericArrayColumnSource() final = default; + + void fillChunk(const RowSequence &rows, Chunk *dest, + BooleanChunk *optionalNullFlags) const final { + typedef typename deephaven::client::chunk::TypeToChunk::type_t chunkType_t; + internal::ColumnSourceImpls::fillChunk(rows, dest, optionalNullFlags, data_); + } + + void fillChunkUnordered(const UInt64Chunk &rowKeys, Chunk *dest, + BooleanChunk *optionalNullFlags) const final { + typedef typename deephaven::client::chunk::TypeToChunk::type_t chunkType_t; + internal::ColumnSourceImpls::fillChunkUnordered(rowKeys, dest, optionalNullFlags, data_); + } + + void fillFromChunk(const Chunk &src, const BooleanChunk *optionalNullFlags, + const RowSequence &rows) final { + typedef typename deephaven::client::chunk::TypeToChunk::type_t chunkType_t; + internal::ColumnSourceImpls::fillFromChunk(src, optionalNullFlags, rows, &data_); + } + + void fillFromChunkUnordered(const Chunk &src, + const BooleanChunk *optionalNullFlags, const UInt64Chunk &rowKeys) final { + typedef typename deephaven::client::chunk::TypeToChunk::type_t chunkType_t; + internal::ColumnSourceImpls::fillFromChunkUnordered(src, optionalNullFlags, + rowKeys, &data_); + } + + void acceptVisitor(ColumnSourceVisitor *visitor) const final { + visitor->visit(*this); + } + +private: + internal::NumericBackingStore data_; +}; + +template +class GenericArrayColumnSource final : public MutableGenericColumnSource, + std::enable_shared_from_this> { + struct Private { + }; + typedef deephaven::client::chunk::BooleanChunk BooleanChunk; + typedef deephaven::client::chunk::Chunk Chunk; + typedef deephaven::client::chunk::UInt64Chunk UInt64Chunk; + typedef deephaven::client::container::RowSequence RowSequence; + +public: + static std::shared_ptr create() { + return std::make_shared(Private()); + } + + explicit GenericArrayColumnSource(Private) {} + ~GenericArrayColumnSource() final = default; + + void fillChunk(const RowSequence &rows, Chunk *dest, BooleanChunk *optionalDestNullFlags) const final { + internal::ColumnSourceImpls::fillChunk(rows, dest, optionalDestNullFlags, data_); + } + + void fillChunkUnordered(const UInt64Chunk &rowKeys, Chunk *dest, + BooleanChunk *optionalDestNullFlags) const final { + internal::ColumnSourceImpls::fillChunkUnordered(rowKeys, dest, + optionalDestNullFlags, data_); + } + + void fillFromChunk(const Chunk &src, const BooleanChunk *optionalSrcNullFlags, + const RowSequence &rows) final { + internal::ColumnSourceImpls::fillFromChunk(src, optionalSrcNullFlags, rows, &data_); + } + + void fillFromChunkUnordered(const Chunk &src, const BooleanChunk *optionalSrcNullFlags, + const UInt64Chunk &rowKeys) final { + internal::ColumnSourceImpls::fillFromChunkUnordered(src, optionalSrcNullFlags, + rowKeys, &data_); + } + + void acceptVisitor(ColumnSourceVisitor *visitor) const final { + visitor->visit(*this); + } + +private: + internal::GenericBackingStore data_; +}; +} // namespace deephaven::client::column diff --git a/cpp-client/deephaven/client/include/public/deephaven/client/column/column_source.h b/cpp-client/deephaven/client/include/public/deephaven/client/column/column_source.h index b1dfa14cc21..94d7543845b 100644 --- a/cpp-client/deephaven/client/include/public/deephaven/client/column/column_source.h +++ b/cpp-client/deephaven/client/include/public/deephaven/client/column/column_source.h @@ -9,6 +9,7 @@ #include #include "deephaven/client/chunk/chunk.h" #include "deephaven/client/container/row_sequence.h" +#include "deephaven/client/types.h" #include "deephaven/client/utility/utility.h" namespace deephaven::client::column { @@ -17,6 +18,7 @@ class ColumnSourceVisitor; // the column source interfaces class ColumnSource { protected: + typedef deephaven::client::chunk::BooleanChunk BooleanChunk; typedef deephaven::client::chunk::Chunk Chunk; typedef deephaven::client::chunk::UInt64Chunk UInt64Chunk; typedef deephaven::client::container::RowSequence RowSequence; @@ -24,11 +26,17 @@ class ColumnSource { public: virtual ~ColumnSource(); - virtual void fillChunk(const RowSequence &rows, Chunk *dest) const = 0; - virtual void fillChunkUnordered(const UInt64Chunk &rowKeys, Chunk *dest) const = 0; + virtual void fillChunk(const RowSequence &rows, Chunk *destData, + BooleanChunk *optionalDestNullFlags) const = 0; + virtual void fillChunkUnordered(const UInt64Chunk &rowKeys, Chunk *dest, + BooleanChunk *optionalDestNullFlags) const = 0; virtual void acceptVisitor(ColumnSourceVisitor *visitor) const = 0; + /** + * Temporary access to some implementation-defined subclass information. Used for debugging and + * until we finalize the API. + */ virtual std::any backdoor() const { return 0; } @@ -38,8 +46,10 @@ class MutableColumnSource : public virtual ColumnSource { public: ~MutableColumnSource() override; - virtual void fillFromChunk(const Chunk &src, const RowSequence &rows) = 0; - virtual void fillFromChunkUnordered(const Chunk &src, const UInt64Chunk &rowKeys) = 0; + virtual void fillFromChunk(const Chunk &src, const BooleanChunk *optionalSrcNullFlags, + const RowSequence &rows) = 0; + virtual void fillFromChunkUnordered(const Chunk &srcData, + const BooleanChunk *optionalSrcNullFlags, const UInt64Chunk &rowKeys) = 0; }; // the per-type interfaces @@ -47,8 +57,9 @@ template class NumericColumnSource : public virtual ColumnSource { }; -// TODO(kosak): it's not obvious to me that String needs to be handled separately. -class StringColumnSource : public virtual ColumnSource { +// the per-type interfaces +template +class GenericColumnSource : public virtual ColumnSource { }; // convenience typedefs @@ -59,157 +70,19 @@ typedef NumericColumnSource Int64ColumnSource; typedef NumericColumnSource FloatColumnSource; typedef NumericColumnSource DoubleColumnSource; +typedef GenericColumnSource BooleanColumnSource; +typedef GenericColumnSource StringColumnSource; +typedef GenericColumnSource DateTimeColumnSource; + // the mutable per-type interfaces template class MutableNumericColumnSource : public NumericColumnSource, public MutableColumnSource { }; -class MutableStringColumnSource : public StringColumnSource, public MutableColumnSource { -}; - template -class NumericArrayColumnSource final : public MutableNumericColumnSource, - std::enable_shared_from_this> { - struct Private {}; - typedef deephaven::client::chunk::Chunk Chunk; - typedef deephaven::client::chunk::UInt64Chunk UInt64Chunk; - typedef deephaven::client::container::RowSequence RowSequence; - -public: - static std::shared_ptr create(); - explicit NumericArrayColumnSource(Private) {} - ~NumericArrayColumnSource() final = default; - - void fillChunk(const RowSequence &rows, Chunk *dest) const final; - void fillChunkUnordered(const UInt64Chunk &rowKeys, Chunk *dest) const final; - void fillFromChunk(const Chunk &src, const RowSequence &rows) final; - void fillFromChunkUnordered(const Chunk &src, const UInt64Chunk &rowKeys) final; - - void acceptVisitor(ColumnSourceVisitor *visitor) const final; - -private: - void ensureSize(size_t size); - - std::vector data_; -}; - -class StringArrayColumnSource final : public MutableStringColumnSource, - std::enable_shared_from_this { - struct Private {}; - typedef deephaven::client::chunk::Chunk Chunk; - typedef deephaven::client::chunk::UInt64Chunk UInt64Chunk; - typedef deephaven::client::container::RowSequence RowSequence; - -public: - static std::shared_ptr create(); - explicit StringArrayColumnSource(Private) {} - ~StringArrayColumnSource() final = default; - - void fillChunk(const RowSequence &rows, Chunk *dest) const final; - void fillChunkUnordered(const UInt64Chunk &rowKeys, Chunk *dest) const final; - void fillFromChunk(const Chunk &src, const RowSequence &rows) final; - void fillFromChunkUnordered(const Chunk &src, const UInt64Chunk &rowKeys) final; - - void acceptVisitor(ColumnSourceVisitor *visitor) const final; - -private: - void ensureSize(size_t size); - - std::vector data_; +class MutableGenericColumnSource : public GenericColumnSource, public MutableColumnSource { }; -template -std::shared_ptr> NumericArrayColumnSource::create() { - return std::make_shared>(Private()); -} - -template -void NumericArrayColumnSource::fillChunk(const RowSequence &rows, Chunk *dest) const { - using deephaven::client::chunk::TypeToChunk; - using deephaven::client::utility::assertLessEq; - using deephaven::client::utility::verboseCast; - typedef typename TypeToChunk::type_t chunkType_t; - - auto *typedDest = verboseCast(dest, DEEPHAVEN_PRETTY_FUNCTION); - // assert rows.size() <= typedDest->size() - assertLessEq(rows.size(), typedDest->size(), "rows.size()", "typedDest->size()", __PRETTY_FUNCTION__); - - size_t destIndex = 0; - auto applyChunk = [this, typedDest, &destIndex](uint64_t begin, uint64_t end) { - // assert end <= data_.size() - assertLessEq(end, data_.size(), "end", "data_.size()", __PRETTY_FUNCTION__); - for (auto current = begin; current != end; ++current) { - typedDest->data()[destIndex] = data_[current]; - ++destIndex; - } - }; - rows.forEachChunk(applyChunk); -} - -template -void NumericArrayColumnSource::fillChunkUnordered(const UInt64Chunk &rowKeys, Chunk *dest) const { - using deephaven::client::chunk::TypeToChunk; - using deephaven::client::utility::assertLessEq; - using deephaven::client::utility::verboseCast; - typedef typename TypeToChunk::type_t chunkType_t; - - auto *typedDest = verboseCast(dest, DEEPHAVEN_PRETTY_FUNCTION); - // assert size <= dest->capacity() - assertLessEq(rowKeys.size(), typedDest->size(), "rowKeys.size()", "typedDest->size()", __PRETTY_FUNCTION__); - - for (size_t i = 0; i < rowKeys.size(); ++i) { - auto srcIndex = rowKeys.data()[i]; - assertLessEq(srcIndex, data_.size(), "srcIndex", "data_.size()", DEEPHAVEN_PRETTY_FUNCTION); - typedDest->data()[i] = this->data_[srcIndex]; - } -} - -template -void NumericArrayColumnSource::fillFromChunk(const Chunk &src, const RowSequence &rows) { - using deephaven::client::chunk::TypeToChunk; - using deephaven::client::utility::assertLessEq; - using deephaven::client::utility::verboseCast; - typedef typename TypeToChunk::type_t chunkType_t; - - const auto *typedSrc = verboseCast(&src, DEEPHAVEN_PRETTY_FUNCTION); - assertLessEq(rows.size(), typedSrc->size(), "rows.size()", "src.size()", __PRETTY_FUNCTION__); - - const auto *srcp = typedSrc->data(); - auto applyChunk = [this, &srcp](uint64_t begin, uint64_t end) { - ensureSize(end); - for (auto current = begin; current != end; ++current) { - data_[current] = *srcp++; - } - }; - rows.forEachChunk(applyChunk); -} - -template -void NumericArrayColumnSource::fillFromChunkUnordered(const Chunk &src, - const UInt64Chunk &rowKeys) { - using deephaven::client::chunk::TypeToChunk; - using deephaven::client::utility::assertLessEq; - using deephaven::client::utility::verboseCast; - typedef typename TypeToChunk::type_t chunkType_t; - - const auto *typedSrc = verboseCast(&src, DEEPHAVEN_PRETTY_FUNCTION); - // assert rowKeys.size() <= src.capacity() - assertLessEq(typedSrc->size(), rowKeys.size(), "src.size()", "rowKeys.size()", __PRETTY_FUNCTION__); - - for (size_t i = 0; i < typedSrc->size(); ++i) { - auto destIndex = rowKeys.data()[i]; - ensureSize(destIndex + 1); - data_[destIndex] = typedSrc->data()[i]; - } -} - -template -void NumericArrayColumnSource::ensureSize(size_t size) { - if (size > data_.size()) { - data_.resize(size); - } -} - class ColumnSourceVisitor { public: virtual void visit(const Int8ColumnSource &) = 0; @@ -218,11 +91,8 @@ class ColumnSourceVisitor { virtual void visit(const Int64ColumnSource &) = 0; virtual void visit(const FloatColumnSource &) = 0; virtual void visit(const DoubleColumnSource &) = 0; + virtual void visit(const BooleanColumnSource &) = 0; virtual void visit(const StringColumnSource &) = 0; + virtual void visit(const DateTimeColumnSource &) = 0; }; - -template -void NumericArrayColumnSource::acceptVisitor(ColumnSourceVisitor *visitor) const { - visitor->visit(*this); -} } // namespace deephaven::client::column diff --git a/cpp-client/deephaven/client/include/public/deephaven/client/container/row_sequence.h b/cpp-client/deephaven/client/include/public/deephaven/client/container/row_sequence.h index 3fe3b0cdaf3..2d098bf2446 100644 --- a/cpp-client/deephaven/client/include/public/deephaven/client/container/row_sequence.h +++ b/cpp-client/deephaven/client/include/public/deephaven/client/container/row_sequence.h @@ -20,7 +20,7 @@ class RowSequence { virtual ~RowSequence(); - virtual std::shared_ptr getRowSequenceIterator() const = 0; + RowSequenceIterator getRowSequenceIterator() const; virtual std::shared_ptr take(size_t size) const = 0; virtual std::shared_ptr drop(size_t size) const = 0; @@ -37,9 +37,21 @@ class RowSequence { }; class RowSequenceIterator { + static constexpr size_t chunkSize = 8192; + struct Private {}; public: - virtual ~RowSequenceIterator(); - virtual bool tryGetNext(uint64_t *result) = 0; + explicit RowSequenceIterator(std::shared_ptr rowSequence); + RowSequenceIterator(RowSequenceIterator &&other) noexcept; + ~RowSequenceIterator(); + bool tryGetNext(uint64_t *result); + +private: + void refillRanges(); + + std::shared_ptr residual_; + std::vector> ranges_; + size_t rangeIndex_ = 0; + uint64_t offset_ = 0; }; class RowSequenceBuilder { diff --git a/cpp-client/deephaven/client/include/public/deephaven/client/table/table.h b/cpp-client/deephaven/client/include/public/deephaven/client/table/table.h index 7482c0864ff..da83426efcf 100644 --- a/cpp-client/deephaven/client/include/public/deephaven/client/table/table.h +++ b/cpp-client/deephaven/client/include/public/deephaven/client/table/table.h @@ -10,6 +10,46 @@ #include "deephaven/client/container/row_sequence.h" namespace deephaven::client::table { +class Table; + +namespace internal { +class TableStreamAdaptor { + typedef deephaven::client::container::RowSequence RowSequence; +public: + TableStreamAdaptor(const Table &table, + std::vector> rowSequences, bool wantHeaders, bool wantRowNumbers, + bool highlightCells) : table_(table), rowSequences_(std::move(rowSequences)), + wantHeaders_(wantHeaders), wantRowNumbers_(wantRowNumbers), highlightCells_(highlightCells) {} + TableStreamAdaptor(const TableStreamAdaptor &) = delete; + TableStreamAdaptor &operator=(const TableStreamAdaptor &) = delete; + ~TableStreamAdaptor() = default; + +private: + const Table &table_; + std::vector> rowSequences_; + bool wantHeaders_ = false; + bool wantRowNumbers_ = false; + bool highlightCells_ = false; + + friend std::ostream &operator<<(std::ostream &s, const TableStreamAdaptor &o); +}; +} // namespace internal + +class Schema { +public: + explicit Schema(std::vector>> columns); + Schema(Schema &&other) noexcept; + Schema &operator=(Schema &&other) noexcept; + ~Schema(); + + const std::vector>> &columns() const { + return columns_; + } + +private: + std::vector>> columns_; +}; + class Table { protected: typedef deephaven::client::column::ColumnSource ColumnSource; @@ -22,7 +62,19 @@ class Table { virtual std::shared_ptr getRowSequence() const = 0; virtual std::shared_ptr getColumn(size_t columnIndex) const = 0; + std::shared_ptr getColumn(std::string_view name, bool strict) const; + virtual size_t numRows() const = 0; virtual size_t numColumns() const = 0; + + virtual const Schema &schema() const = 0; + + internal::TableStreamAdaptor stream(bool wantHeaders, bool wantRowNumbers) const; + + internal::TableStreamAdaptor stream(bool wantHeaders, bool wantRowNumbers, + std::shared_ptr rowSequence) const; + + internal::TableStreamAdaptor stream(bool wantHeaders, bool wantRowNumbers, + std::vector> rowSequences) const; }; -} // namespace deephaven::client::highlevel::table +} // namespace deephaven::client::table diff --git a/cpp-client/deephaven/client/include/public/deephaven/client/ticking.h b/cpp-client/deephaven/client/include/public/deephaven/client/ticking.h index 10ff74e4eb9..fbbbe491a85 100644 --- a/cpp-client/deephaven/client/include/public/deephaven/client/ticking.h +++ b/cpp-client/deephaven/client/include/public/deephaven/client/ticking.h @@ -13,111 +13,55 @@ #include "immer/flex_vector.hpp" namespace deephaven::client { -class ClassicTickingUpdate; -class ImmerTickingUpdate; +class TickingUpdate; class TickingCallback : public deephaven::client::utility::FailureCallback { public: - /** - * @param update An update message which describes the changes (removes, adds, modifies) that - * transform the previous version of the table to the new version. This class is *shared* with - * the caller and so the receiving code will block the caller while it is using it. - */ - virtual void onTick(const ClassicTickingUpdate &update) = 0; /** * @param update An update message which describes the changes (removes, adds, modifies) that * transform the previous version of the table to the new version. This class is threadsafe and * can be kept around for an arbitrary amount of time. On the other hand, it probably should be * processed and discard quickly so that the underlying resources can be reused. */ - virtual void onTick(ImmerTickingUpdate update) = 0; + virtual void onTick(TickingUpdate update) = 0; }; -class ClassicTickingUpdate final { +class TickingUpdate final { protected: - typedef deephaven::client::chunk::UInt64Chunk UInt64Chunk; - typedef deephaven::client::column::ColumnSource ColumnSource; typedef deephaven::client::container::RowSequence RowSequence; typedef deephaven::client::table::Table Table; public: - ClassicTickingUpdate(std::shared_ptr removedRowsKeySpace, - UInt64Chunk removedRowsIndexSpace, - std::shared_ptr addedRowsKeySpace, - UInt64Chunk addedRowsIndexSpace, - std::vector> modifiedRowsKeySpace, - std::vector modifiedRowsIndexSpace, - std::shared_ptr currentTableKeySpace, - std::shared_ptr
currentTableIndexSpace); - ClassicTickingUpdate(ClassicTickingUpdate &&other) noexcept; - ClassicTickingUpdate &operator=(ClassicTickingUpdate &&other) noexcept; - ~ClassicTickingUpdate(); + TickingUpdate(std::shared_ptr
prev, + std::shared_ptr removedRows, std::shared_ptr
afterRemoves, + std::shared_ptr addedRows, std::shared_ptr
afterAdds, + std::vector> modifiedRows, std::shared_ptr
afterModifies); + TickingUpdate(TickingUpdate &&other) noexcept; + TickingUpdate &operator=(TickingUpdate &&other) noexcept; + ~TickingUpdate(); - const std::shared_ptr &removedRowsKeySpace() const { return removedRowsKeySpace_; } - const UInt64Chunk &removedRowsIndexSpace() const { return removedRowsIndexSpace_; } - const std::shared_ptr &addedRowsKeySpace() const { return addedRowsKeySpace_; } - const UInt64Chunk &addedRowsIndexSpace() const { return addedRowsIndexSpace_; } - const std::vector> &modifiedRowsKeySpace() const { return modifiedRowsKeySpace_; } - const std::vector &modifiedRowsIndexSpace() const { return modifiedRowsIndexSpace_; } - const std::shared_ptr
¤tTableKeySpace() const { return currentTableKeySpace_; } - const std::shared_ptr
¤tTableIndexSpace() const { return currentTableIndexSpace_; } - -private: - // In the pre-shift key space - std::shared_ptr removedRowsKeySpace_; - // In the pre-shift index space - UInt64Chunk removedRowsIndexSpace_; - // In the post-shift key space - std::shared_ptr addedRowsKeySpace_; - // In the post-shift index space - UInt64Chunk addedRowsIndexSpace_; - // In the post-shift key space - std::vector> modifiedRowsKeySpace_; - // In the post-shift index space - std::vector modifiedRowsIndexSpace_; + const std::shared_ptr
&prev() const { return prev_; } - std::shared_ptr
currentTableKeySpace_; - std::shared_ptr
currentTableIndexSpace_; -}; + const std::shared_ptr
&beforeRemoves() const { return prev_; } + const std::shared_ptr &removedRows() const { return removedRows_; } + const std::shared_ptr
&afterRemoves() const { return afterRemoves_; } -class ImmerTickingUpdate final { -protected: - typedef deephaven::client::container::RowSequence RowSequence; - typedef deephaven::client::table::Table Table; + const std::shared_ptr
&beforeAdds() const { return afterRemoves_; } + const std::shared_ptr &addedRows() const { return addedRows_; } + const std::shared_ptr
&afterAdds() const { return afterAdds_; } -public: - ImmerTickingUpdate(std::shared_ptr
beforeRemoves, - std::shared_ptr
beforeModifies, - std::shared_ptr
current, - std::shared_ptr removed, - std::vector> modified, - std::shared_ptr added); - ImmerTickingUpdate(ImmerTickingUpdate &&other) noexcept; - ImmerTickingUpdate &operator=(ImmerTickingUpdate &&other) noexcept; - ~ImmerTickingUpdate(); + const std::shared_ptr
&beforeModifies() const { return afterAdds_; } + const std::vector> &modifiedRows() const { return modifiedRows_; } + const std::shared_ptr
&afterModifies() const { return afterModifies_; } - // Note: the table is flat. - const std::shared_ptr
&beforeRemoves() const { return beforeRemoves_; } - // Note: the table is flat. - const std::shared_ptr
&beforeModifies() const { return beforeModifies_; } - // Note: the table is flat. - const std::shared_ptr
¤t() const { return current_; } - // In the key space of 'prevTable' - const std::shared_ptr &removed() const { return removed_; } - // In the key space of 'current' - const std::vector> &modified() const { return modified_; } - // In the key space of 'current' - const std::shared_ptr &added() const { return added_; } + const std::shared_ptr
¤t() const { return afterModifies_; } private: - std::shared_ptr
beforeRemoves_; - std::shared_ptr
beforeModifies_; - std::shared_ptr
current_; - // In the key space of 'beforeRemoves_' - std::shared_ptr removed_; - // In the key space of beforeModifies_ and current_, which have the same key space. - // Old values are in beforeModifies_; new values are in current_. - std::vector> modified_; - // In the key space of current_. - std::shared_ptr added_; + std::shared_ptr
prev_; + std::shared_ptr removedRows_; + std::shared_ptr
afterRemoves_; + std::shared_ptr addedRows_; + std::shared_ptr
afterAdds_; + std::vector> modifiedRows_; + std::shared_ptr
afterModifies_; }; } // namespace deephaven::client diff --git a/cpp-client/deephaven/client/include/public/deephaven/client/types.h b/cpp-client/deephaven/client/include/public/deephaven/client/types.h index 0228ffcbb5b..e315e25dd7a 100644 --- a/cpp-client/deephaven/client/include/public/deephaven/client/types.h +++ b/cpp-client/deephaven/client/include/public/deephaven/client/types.h @@ -4,9 +4,9 @@ #pragma once #include +#include #include #include -#include namespace deephaven::client { class DeephavenConstants { @@ -54,6 +54,39 @@ class DeephavenConstants { static constexpr const int64_t MAX_LONG = std::numeric_limits::max(); }; +template +struct DeephavenConstantsForType {}; + +template<> +struct DeephavenConstantsForType { + static constexpr const int8_t NULL_VALUE = DeephavenConstants::NULL_BYTE; +}; + +template<> +struct DeephavenConstantsForType { + static constexpr const int16_t NULL_VALUE = DeephavenConstants::NULL_SHORT; +}; + +template<> +struct DeephavenConstantsForType { + static constexpr const int32_t NULL_VALUE = DeephavenConstants::NULL_INT; +}; + +template<> +struct DeephavenConstantsForType { + static constexpr const int64_t NULL_VALUE = DeephavenConstants::NULL_LONG; +}; + +template<> +struct DeephavenConstantsForType { + static constexpr const float NULL_VALUE = DeephavenConstants::NULL_FLOAT; +}; + +template<> +struct DeephavenConstantsForType { + static constexpr const double NULL_VALUE = DeephavenConstants::NULL_DOUBLE; +}; + /** * The Deephaven DateTime type. Records nanoseconds relative to the epoch (January 1, 1970) UTC. * Times before the epoch can be represented with negative nanosecond values. diff --git a/cpp-client/deephaven/client/include/public/deephaven/client/utility/utility.h b/cpp-client/deephaven/client/include/public/deephaven/client/utility/utility.h index 14ec0496b20..a260f8d6286 100644 --- a/cpp-client/deephaven/client/include/public/deephaven/client/utility/utility.h +++ b/cpp-client/deephaven/client/include/public/deephaven/client/utility/utility.h @@ -25,9 +25,6 @@ std::vector makeReservedVector(size_t n) { return v; } -void assertLessEq(size_t lhs, size_t rhs, std::string_view context, std::string_view lhsName, - std::string_view rhsName); - // A more efficient ostringstream that also allows you to grab the internal buffer if you want it. // Or, if you don't want to use the internal buffer, it allows you to provide your own. class SimpleOstringstream final : private std::basic_streambuf, public std::ostream { @@ -214,7 +211,7 @@ constexpr std::string_view getTypeName() { } template -DESTP verboseCast(SRCP ptr, std::string_view caller) { +DESTP verboseCast(const DebugInfo &debugInfo, SRCP ptr) { using deephaven::client::utility::stringf; auto *typedPtr = dynamic_cast(ptr); @@ -223,21 +220,19 @@ DESTP verboseCast(SRCP ptr, std::string_view caller) { } typedef decltype(*std::declval()) destType_t; auto message = stringf("%o: Expected type %o. Got type %o", - caller, getTypeName(), typeid(*ptr).name()); + debugInfo, getTypeName(), typeid(*ptr).name()); throw std::runtime_error(message); } -/** - * TODO(kosak): Do something else here. Maybe. - */ -template -void assertLessEq(const T &lhs, const T &rhs, std::string_view lhsText, std::string_view rhsText, - std::string_view func) { - if (lhs <= rhs) { +namespace internal { +void trueOrThrowHelper(const DebugInfo &debugInfo); +} // namespace internal + +inline void trueOrThrow(const DebugInfo &debugInfo, bool value) { + if (value) { return; } - throw std::runtime_error(stringf("assertion failed: %o: %o <= %o (%o <= %o)", func, lhs, rhs, - lhsText, rhsText)); + internal::trueOrThrowHelper(debugInfo); } /** diff --git a/cpp-client/deephaven/client/src/chunk/chunk_filler.cc b/cpp-client/deephaven/client/src/chunk/chunk_filler.cc index b49c2cb3e20..999ab8de090 100644 --- a/cpp-client/deephaven/client/src/chunk/chunk_filler.cc +++ b/cpp-client/deephaven/client/src/chunk/chunk_filler.cc @@ -3,48 +3,85 @@ */ #include "deephaven/client/chunk/chunk_filler.h" +#include "deephaven/client/arrowutil/arrow_visitors.h" +#include "deephaven/client/arrowutil/arrow_value_converter.h" #include "deephaven/client/container/row_sequence.h" #include "deephaven/client/impl/util.h" +#include "deephaven/client/types.h" #include "deephaven/client/utility/utility.h" +using deephaven::client::arrowutil::isNumericType; +using deephaven::client::arrowutil::ArrowValueConverter; +using deephaven::client::container::RowSequence; using deephaven::client::utility::okOrThrow; using deephaven::client::utility::stringf; using deephaven::client::utility::verboseCast; -using deephaven::client::container::RowSequence; namespace deephaven::client::chunk { namespace { struct Visitor final : arrow::ArrayVisitor { - Visitor(const RowSequence &keys, Chunk *dest) : keys_(keys), dest_(dest) {} + Visitor(const RowSequence &keys, Chunk *destData, BooleanChunk *optionalDestNullFlags) : + keys_(keys), destData_(destData), optionalDestNullFlags_(optionalDestNullFlags) {} + + arrow::Status Visit(const arrow::Int8Array &array) final { + return fillChunk(array); + } + + arrow::Status Visit(const arrow::Int16Array &array) final { + return fillChunk(array); + } arrow::Status Visit(const arrow::Int32Array &array) final { - return fillNumericChunk(array); + return fillChunk(array); } arrow::Status Visit(const arrow::Int64Array &array) final { - return fillNumericChunk(array); + return fillChunk(array); } - arrow::Status Visit(const arrow::UInt64Array &array) final { - return fillNumericChunk(array); + arrow::Status Visit(const arrow::FloatArray &array) final { + return fillChunk(array); } arrow::Status Visit(const arrow::DoubleArray &array) final { - return fillNumericChunk(array); + return fillChunk(array); + } + + arrow::Status Visit(const arrow::StringArray &array) final { + return fillChunk(array); + } + + arrow::Status Visit(const arrow::BooleanArray &array) final { + return fillChunk(array); } template - arrow::Status fillNumericChunk(const TArrowAray &array) { - auto *typedDest = verboseCast*>(dest_, DEEPHAVEN_PRETTY_FUNCTION); + arrow::Status fillChunk(const TArrowAray &array) { + auto *typedDest = verboseCast*>(DEEPHAVEN_EXPR_MSG(destData_)); + auto *destDatap = typedDest->data(); + auto *destNullp = optionalDestNullFlags_ != nullptr ? optionalDestNullFlags_->data() : nullptr; checkSize(typedDest->size()); size_t destIndex = 0; - auto copyChunk = [&destIndex, &array, typedDest](uint64_t begin, uint64_t end) { - for (auto current = begin; current != end; ++current) { - if (array.IsNull(current)) { - throw std::runtime_error(DEEPHAVEN_DEBUG_MSG("Not handling nulls yet")); + + auto copyChunk = + [&array, destDatap, destNullp, &destIndex](uint64_t begin, uint64_t end) { + auto beginp = array.begin() + begin; + auto endp = array.begin() + end; + for (auto currentp = beginp; currentp != endp; ++currentp) { + auto optValue = *currentp; + bool isNull = !optValue.has_value(); + T value = T(); + if (isNull) { + if constexpr(isNumericType()) { + value = deephaven::client::DeephavenConstantsForType::NULL_VALUE; + } + } else { + ArrowValueConverter::convert(*optValue, &value); + } + destDatap[destIndex] = std::move(value); + if (destNullp != nullptr) { + destNullp[destIndex] = isNull; } - auto val = array.Value((int64_t)current); - typedDest->data()[destIndex] = val; ++destIndex; } }; @@ -60,12 +97,14 @@ struct Visitor final : arrow::ArrayVisitor { } const RowSequence &keys_; - Chunk *const dest_; + Chunk *const destData_ = nullptr; + BooleanChunk *const optionalDestNullFlags_ = nullptr; }; } // namespace -void ChunkFiller::fillChunk(const arrow::Array &src, const RowSequence &keys, Chunk *dest) { - Visitor visitor(keys, dest); +void ChunkFiller::fillChunk(const arrow::Array &src, const RowSequence &keys, Chunk *destData, + BooleanChunk *optionalDestNullFlags) { + Visitor visitor(keys, destData, optionalDestNullFlags); okOrThrow(DEEPHAVEN_EXPR_MSG(src.Accept(&visitor))); } } // namespace deephaven::client::chunk diff --git a/cpp-client/deephaven/client/src/chunk/chunk_maker.cc b/cpp-client/deephaven/client/src/chunk/chunk_maker.cc index a8287ee14a9..16e6faf46e6 100644 --- a/cpp-client/deephaven/client/src/chunk/chunk_maker.cc +++ b/cpp-client/deephaven/client/src/chunk/chunk_maker.cc @@ -6,6 +6,7 @@ #include "deephaven/client/column/column_source.h" using deephaven::client::column::ColumnSourceVisitor; +using deephaven::client::column::DateTimeColumnSource; using deephaven::client::column::DoubleColumnSource; using deephaven::client::column::FloatColumnSource; using deephaven::client::column::Int8ColumnSource; @@ -43,10 +44,18 @@ struct Visitor final : ColumnSourceVisitor { result_ = DoubleChunk::create(chunkSize_); } + void visit(const column::BooleanColumnSource &source) final { + result_ = BooleanChunk::create(chunkSize_); + } + void visit(const StringColumnSource &source) final { result_ = StringChunk::create(chunkSize_); } + void visit(const DateTimeColumnSource &source) final { + result_ = DateTimeChunk::create(chunkSize_); + } + size_t chunkSize_; AnyChunk result_; }; diff --git a/cpp-client/deephaven/client/src/client.cc b/cpp-client/deephaven/client/src/client.cc index 1afeb93bba8..457b2beb19d 100644 --- a/cpp-client/deephaven/client/src/client.cc +++ b/cpp-client/deephaven/client/src/client.cc @@ -522,8 +522,8 @@ std::shared_ptr TableHandle::getFlightStreamR } std::shared_ptr TableHandle::subscribe( - std::shared_ptr callback, bool wantImmer) { - return impl_->subscribe(std::move(callback), wantImmer); + std::shared_ptr callback) { + return impl_->subscribe(std::move(callback)); } void TableHandle::unsubscribe(std::shared_ptr callback) { diff --git a/cpp-client/deephaven/client/src/column/array_column_source.cc b/cpp-client/deephaven/client/src/column/array_column_source.cc new file mode 100644 index 00000000000..7ea15d1f71c --- /dev/null +++ b/cpp-client/deephaven/client/src/column/array_column_source.cc @@ -0,0 +1,15 @@ +/** + * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending + */ +#include "deephaven/client/column/array_column_source.h" + +namespace deephaven::client::column { + +using deephaven::client::utility::trueOrThrow; + +namespace internal { +void BackingStoreBase::assertIndexValid(size_t index) const { + trueOrThrow(DEEPHAVEN_EXPR_MSG(index < capacity_)); +} +} // namespace internal +} // namespace deephaven::client::column diff --git a/cpp-client/deephaven/client/src/container/row_sequence.cc b/cpp-client/deephaven/client/src/container/row_sequence.cc index a1b20478da9..be437a5b256 100644 --- a/cpp-client/deephaven/client/src/container/row_sequence.cc +++ b/cpp-client/deephaven/client/src/container/row_sequence.cc @@ -7,24 +7,47 @@ using deephaven::client::utility::stringf; namespace deephaven::client::container { +namespace { +class SequentialRowSequence final : public RowSequence { +public: + static std::shared_ptr create(uint64_t begin, uint64_t end); + + SequentialRowSequence(uint64_t begin, uint64_t end) : begin_(begin), end_(end) {} + + std::shared_ptr take(size_t size) const final; + std::shared_ptr drop(size_t size) const final; + void forEachChunk(const std::function &f) const final; + + size_t size() const final { + return end_ - begin_; + } + +private: + uint64_t begin_ = 0; + uint64_t end_ = 0; +}; +} // namespace + std::shared_ptr RowSequence::createEmpty() { return RowSequenceBuilder().build(); } std::shared_ptr RowSequence::createSequential(uint64_t begin, uint64_t end) { - RowSequenceBuilder builder; - builder.addRange(begin, end); - return builder.build(); + return SequentialRowSequence::create(begin, end); } RowSequence::~RowSequence() = default; +RowSequenceIterator RowSequence::getRowSequenceIterator() const { + return RowSequenceIterator(drop(0)); +} + std::ostream &operator<<(std::ostream &s, const RowSequence &o) { s << '['; auto iter = o.getRowSequenceIterator(); const char *sep = ""; uint64_t item; - while (iter->tryGetNext(&item)) { + while (iter.tryGetNext(&item)) { s << sep << item; sep = ", "; } @@ -32,6 +55,46 @@ std::ostream &operator<<(std::ostream &s, const RowSequence &o) { return s; } +RowSequenceIterator::RowSequenceIterator(std::shared_ptr rowSequence) : + residual_(std::move(rowSequence)) {} +RowSequenceIterator::RowSequenceIterator(RowSequenceIterator &&other) noexcept = default; +RowSequenceIterator::~RowSequenceIterator() = default; + +bool RowSequenceIterator::tryGetNext(uint64_t *result) { + while (true) { + if (rangeIndex_ == ranges_.size()) { + rangeIndex_ = 0; + refillRanges(); + if (ranges_.empty()) { + return false; + } + continue; + } + + const auto &range = ranges_[rangeIndex_]; + auto rangeSize = range.second - range.first; + if (offset_ == rangeSize) { + ++rangeIndex_; + offset_ = 0; + continue; + } + + *result = range.first + offset_; + ++offset_; + return true; + } +} + +void RowSequenceIterator::refillRanges() { + auto thisChunk = residual_->take(chunkSize); + residual_ = residual_->drop(chunkSize); + ranges_.clear(); + auto addRange = [this](uint64_t beginKey, uint64_t endKey) { + ranges_.emplace_back(beginKey, endKey); + }; + thisChunk->forEachChunk(addRange); +} + namespace { class MyRowSequence final : public RowSequence { // begin->end @@ -40,12 +103,11 @@ class MyRowSequence final : public RowSequence { MyRowSequence(std::shared_ptr ranges, ranges_t::const_iterator beginp, size_t entryOffset, size_t size); ~MyRowSequence() final = default; - std::shared_ptr getRowSequenceIterator() const final; std::shared_ptr take(size_t size) const final; std::shared_ptr drop(size_t size) const final; - void forEachChunk(const std::function &f) const final; + void forEachChunk(const std::function &f) const final; size_t size() const final { return size_; @@ -57,30 +119,15 @@ class MyRowSequence final : public RowSequence { size_t entryOffset_ = 0; size_t size_ = 0; }; - -class MyRowSequenceIterator final : public RowSequenceIterator { - typedef std::map ranges_t; -public: - MyRowSequenceIterator(std::shared_ptr ranges, ranges_t::const_iterator currentp, - size_t currentOffset, size_t size); - ~MyRowSequenceIterator() final = default; - bool tryGetNext(uint64_t *result) final; - -private: - std::shared_ptr ranges_; - ranges_t::const_iterator current_; - size_t currentOffset_ = 0; - size_t size_ = 0; -}; } // namespace -RowSequenceIterator::~RowSequenceIterator() = default; RowSequenceBuilder::RowSequenceBuilder() = default; RowSequenceBuilder::~RowSequenceBuilder() = default; void RowSequenceBuilder::addRange(uint64_t begin, uint64_t end) { if (begin > end) { - throw std::runtime_error(stringf("Malformed range [%o,%o)", begin, end)); + auto message = stringf("Malformed range [%o,%o)", begin, end); + throw std::runtime_error(DEEPHAVEN_DEBUG_MSG(message)); } if (begin == end) { @@ -180,10 +227,6 @@ MyRowSequence::MyRowSequence(std::shared_ptr ranges, ranges_t::const_i size_t entryOffset, size_t size) : ranges_(std::move(ranges)), beginp_(beginp), entryOffset_(entryOffset), size_(size) {} -std::shared_ptr MyRowSequence::getRowSequenceIterator() const { - return std::make_shared(ranges_, beginp_, entryOffset_, size_); -} - std::shared_ptr MyRowSequence::take(size_t size) const { auto newSize = std::min(size, size_); return std::make_shared(ranges_, beginp_, entryOffset_, newSize); @@ -232,24 +275,22 @@ void MyRowSequence::forEachChunk(const std::function ranges, - ranges_t::const_iterator current, size_t currentOffset, size_t size) : - ranges_(std::move(ranges)), current_(current), currentOffset_(currentOffset), size_(size) {} - -bool MyRowSequenceIterator::tryGetNext(uint64_t *result) { - while (size_ != 0) { - auto entrySize = current_->second - current_->first; - if (currentOffset_ < entrySize) { - *result = current_->first + currentOffset_; - ++currentOffset_; - --size_; - return true; - } - currentOffset_ = 0; - ++current_; - } - return false; +std::shared_ptr SequentialRowSequence::create(uint64_t begin, uint64_t end) { + return std::make_shared(begin, end); +} + +std::shared_ptr SequentialRowSequence::take(size_t size) const { + auto sizeToUse = std::min(size, this->size()); + return create(begin_, begin_ + sizeToUse); +} + +std::shared_ptr SequentialRowSequence::drop(size_t size) const { + auto sizeToUse = std::min(size, this->size()); + return create(begin_ + sizeToUse, end_); +} + +void SequentialRowSequence::forEachChunk(const std::function &f) const { + f(begin_, end_); } } // namespace } // namespace deephaven::client::container - diff --git a/cpp-client/deephaven/client/src/impl/table_handle_impl.cc b/cpp-client/deephaven/client/src/impl/table_handle_impl.cc index 22e917f441e..439f63981d4 100644 --- a/cpp-client/deephaven/client/src/impl/table_handle_impl.cc +++ b/cpp-client/deephaven/client/src/impl/table_handle_impl.cc @@ -340,7 +340,7 @@ std::shared_ptr TableHandleImpl::asOfJoin(AsOfJoinTablesRequest } std::shared_ptr TableHandleImpl::subscribe( - std::shared_ptr callback, bool wantImmer) { + std::shared_ptr callback) { // On the flight executor thread, we invoke DoExchange (waiting for a successful response). // We wait for that response here. That makes the first part of this call synchronous. If there // is an error in the DoExchange invocation, the caller will get an exception here. The @@ -348,7 +348,7 @@ std::shared_ptr TableHandleImpl::subscribe( // parsing of all the replies) is done on a newly-created thread dedicated to that job. auto colDefs = lazyState_->getColumnDefinitions(); auto handle = startSubscribeThread(managerImpl_->server(), managerImpl_->flightExecutor().get(), - colDefs, ticket_, std::move(callback), wantImmer); + colDefs, ticket_, std::move(callback)); subscriptions_.insert(handle); return handle; diff --git a/cpp-client/deephaven/client/src/subscription/batch_parser.cc b/cpp-client/deephaven/client/src/subscription/batch_parser.cc index 804fb1063e8..a5bec484f24 100644 --- a/cpp-client/deephaven/client/src/subscription/batch_parser.cc +++ b/cpp-client/deephaven/client/src/subscription/batch_parser.cc @@ -13,13 +13,12 @@ using deephaven::client::utility::stringf; namespace deephaven::client::subscription { // Processes all of the adds in this add batch. Will invoke (numAdds - 1) additional calls to GetNext(). void BatchParser::parseBatches( - const ColumnDefinitions &colDefs, + size_t expectedNumCols, size_t numBatches, bool allowInconsistentColumnSizes, arrow::flight::FlightStreamReader *fsr, arrow::flight::FlightStreamChunk *flightStreamChunk, const std::function> &)> &callback) { - auto colDefsSize = colDefs.vec().size(); if (numBatches == 0) { return; } @@ -27,22 +26,20 @@ void BatchParser::parseBatches( while (true) { const auto &srcCols = flightStreamChunk->data->columns(); auto ncols = srcCols.size(); - if (ncols != colDefsSize) { - throw std::runtime_error(stringf("Received %o columns, but my table has %o columns", ncols, - colDefsSize)); + if (ncols != expectedNumCols) { + auto message = stringf("Expected %o columns, got %o", expectedNumCols, ncols); + throw std::runtime_error(DEEPHAVEN_DEBUG_MSG(message)); } if (!allowInconsistentColumnSizes) { auto numRows = srcCols[0]->length(); for (size_t i = 1; i < ncols; ++i) { const auto &srcColArrow = *srcCols[i]; - // I think you do not want this check for the modify case. When you are parsing modify - // messages, the columns may indeed be of different sizes. if (srcColArrow.length() != numRows) { auto message = stringf( "Inconsistent column lengths: Column 0 has %o rows, but column %o has %o rows", numRows, i, srcColArrow.length()); - throw std::runtime_error(message); + throw std::runtime_error(DEEPHAVEN_DEBUG_MSG(message)); } } } diff --git a/cpp-client/deephaven/client/src/subscription/classic_table_state.cc b/cpp-client/deephaven/client/src/subscription/classic_table_state.cc index 6fd2aa4141b..48a2f8fb4b2 100644 --- a/cpp-client/deephaven/client/src/subscription/classic_table_state.cc +++ b/cpp-client/deephaven/client/src/subscription/classic_table_state.cc @@ -4,20 +4,28 @@ #include "deephaven/client/subscription/classic_table_state.h" #include +#include "deephaven/client/arrowutil/arrow_visitors.h" #include "deephaven/client/chunk/chunk_filler.h" #include "deephaven/client/chunk/chunk_maker.h" +#include "deephaven/client/column/column_source.h" +#include "deephaven/client/column/array_column_source.h" #include "deephaven/client/container/row_sequence.h" #include "deephaven/client/subscription/shift_processor.h" #include "deephaven/client/utility/utility.h" +using deephaven::client::arrowutil::ArrowTypeVisitor; +using deephaven::client::arrowutil::isNumericType; +using deephaven::client::chunk::BooleanChunk; using deephaven::client::chunk::ChunkFiller; using deephaven::client::chunk::ChunkMaker; using deephaven::client::chunk::UInt64Chunk; using deephaven::client::column::ColumnSource; +using deephaven::client::column::GenericArrayColumnSource; using deephaven::client::column::MutableColumnSource; using deephaven::client::column::NumericArrayColumnSource; using deephaven::client::container::RowSequence; using deephaven::client::container::RowSequenceBuilder; +using deephaven::client::table::Schema; using deephaven::client::table::Table; using deephaven::client::utility::ColumnDefinitions; using deephaven::client::utility::makeReservedVector; @@ -34,7 +42,7 @@ makeColumnSources(const ColumnDefinitions &colDefs); class TableView final : public Table { public: - TableView(std::vector> columns, + TableView(std::shared_ptr schema, std::vector> columns, std::shared_ptr> redirection); ~TableView() final; @@ -50,18 +58,25 @@ class TableView final : public Table { return redirection_->size(); } - size_t numColumns() const override { + size_t numColumns() const final { return columns_.size(); } + const Schema &schema() const final { + return *schema_; + } + private: + std::shared_ptr schema_; std::vector> columns_; std::shared_ptr> redirection_; }; class UnwrappedTableView final : public Table { public: - UnwrappedTableView(std::vector> columns, size_t numRows); + UnwrappedTableView(std::shared_ptr schema, + std::vector> columns, + size_t numRows); ~UnwrappedTableView() final; std::shared_ptr getRowSequence() const final; @@ -78,7 +93,12 @@ class UnwrappedTableView final : public Table { return columns_.size(); } + const Schema &schema() const final { + return *schema_; + } + private: + std::shared_ptr schema_; std::vector> columns_; size_t numRows_ = 0; }; @@ -154,13 +174,14 @@ void ClassicTableState::addData(const std::vector> auto ncols = data.size(); auto nrows = rowsToAddIndexSpace.size(); auto sequentialRows = RowSequence::createSequential(0, nrows); + auto nullFlags = BooleanChunk::create(nrows); for (size_t i = 0; i < ncols; ++i) { const auto &src = *data[i]; auto *dest = columns_[i].get(); auto anyChunk = ChunkMaker::createChunkFor(*dest, nrows); - auto &chunk = anyChunk.unwrap(); - ChunkFiller::fillChunk(src, *sequentialRows, &chunk); - dest->fillFromChunkUnordered(chunk, rowsToAddIndexSpace); + auto &dataChunk = anyChunk.unwrap(); + ChunkFiller::fillChunk(src, *sequentialRows, &dataChunk, &nullFlags); + dest->fillFromChunkUnordered(dataChunk, &nullFlags, rowsToAddIndexSpace); } } @@ -177,11 +198,11 @@ void ClassicTableState::applyShifts(const RowSequence &firstIndex, const RowSequ } std::shared_ptr
ClassicTableState::snapshot() const { - return std::make_shared(columns_, redirection_); + return std::make_shared(schema_, columns_, redirection_); } std::shared_ptr
ClassicTableState::snapshotUnwrapped() const { - return std::make_shared(columns_, redirection_->size()); + return std::make_shared(schema_, columns_, redirection_->size()); } std::vector ClassicTableState::modifyKeys( @@ -237,8 +258,9 @@ void ClassicTableState::modifyData(const std::vectorfillFromChunkUnordered(chunk, rows); + auto nullFlags = BooleanChunk::create(nrows); + ChunkFiller::fillChunk(srcArray, *sequentialRows, &chunk, &nullFlags); + destCol->fillFromChunkUnordered(chunk, &nullFlags, rows); } } @@ -293,20 +315,14 @@ void mapShifter(uint64_t begin, uint64_t end, uint64_t dest, std::map::create(); - return arrow::Status::OK(); - } - - arrow::Status Visit(const arrow::Int64Type &type) final { - columnSource_ = NumericArrayColumnSource::create(); - return arrow::Status::OK(); - } - - arrow::Status Visit(const arrow::DoubleType &type) final { - columnSource_ = NumericArrayColumnSource::create(); - return arrow::Status::OK(); +struct ColumnSourceMaker final { + template + void operator()() { + if constexpr(isNumericType()) { + columnSource_ = NumericArrayColumnSource::create(); + } else { + columnSource_ = GenericArrayColumnSource::create(); + } } std::shared_ptr columnSource_; @@ -315,25 +331,27 @@ struct MyVisitor final : public arrow::TypeVisitor { std::vector> makeColumnSources(const ColumnDefinitions &colDefs) { std::vector> result; for (const auto &[name, arrowType] : colDefs.vec()) { - MyVisitor v; + ArrowTypeVisitor v; okOrThrow(DEEPHAVEN_EXPR_MSG(arrowType->Accept(&v))); - result.push_back(v.columnSource_); + result.push_back(std::move(v.inner().columnSource_)); } return result; } -TableView::TableView(std::vector> columns, - std::shared_ptr> redirection) : columns_(std::move(columns)), - redirection_(std::move(redirection)) {} +TableView::TableView(std::shared_ptr schema, + std::vector> columns, + std::shared_ptr> redirection) : schema_(std::move(schema)), + columns_(std::move(columns)), redirection_(std::move(redirection)) {} TableView::~TableView() = default; std::shared_ptr TableView::getRowSequence() const { throw std::runtime_error("TODO(kosak)"); } -UnwrappedTableView::UnwrappedTableView(std::vector> columns, - size_t numRows) : columns_(std::move(columns)), numRows_(numRows) {} +UnwrappedTableView::UnwrappedTableView(std::shared_ptr schema, + std::vector> columns, size_t numRows) : + schema_(std::move(schema)), columns_(std::move(columns)), numRows_(numRows) {} UnwrappedTableView::~UnwrappedTableView() = default; std::shared_ptr UnwrappedTableView::getRowSequence() const { diff --git a/cpp-client/deephaven/client/src/subscription/immer_table_state.cc b/cpp-client/deephaven/client/src/subscription/immer_table_state.cc index e7449a83ff4..f354cbc4e92 100644 --- a/cpp-client/deephaven/client/src/subscription/immer_table_state.cc +++ b/cpp-client/deephaven/client/src/subscription/immer_table_state.cc @@ -13,11 +13,10 @@ #include "deephaven/client/immerutil/abstract_flex_vector.h" #include "deephaven/client/subscription/shift_processor.h" #include "deephaven/client/utility/utility.h" -#include "immer/flex_vector.hpp" -#include "immer/flex_vector_transient.hpp" using deephaven::client::arrowutil::ArrowTypeVisitor; using deephaven::client::arrowutil::ArrowArrayTypeVisitor; +using deephaven::client::arrowutil::isNumericType; using deephaven::client::chunk::Int64Chunk; using deephaven::client::column::ColumnSource; using deephaven::client::container::RowSequence; @@ -25,6 +24,9 @@ using deephaven::client::container::RowSequenceBuilder; using deephaven::client::container::RowSequenceIterator; using deephaven::client::subscription::ShiftProcessor; using deephaven::client::immerutil::AbstractFlexVectorBase; +using deephaven::client::immerutil::GenericAbstractFlexVector; +using deephaven::client::immerutil::NumericAbstractFlexVector; +using deephaven::client::table::Schema; using deephaven::client::table::Table; using deephaven::client::utility::ColumnDefinitions; using deephaven::client::utility::makeReservedVector; @@ -34,24 +36,32 @@ using deephaven::client::utility::stringf; namespace deephaven::client::subscription { namespace { -// void mapShifter(int64_t start, int64_t endInclusive, int64_t dest, std::map *zm); class MyTable final : public Table { public: - explicit MyTable(std::vector> sources, size_t numRows); + explicit MyTable(std::shared_ptr schema, + std::vector> sources, size_t numRows); ~MyTable() final; std::shared_ptr getRowSequence() const final; + std::shared_ptr getColumn(size_t columnIndex) const final { return sources_[columnIndex]; } + size_t numRows() const final { return numRows_; } + size_t numColumns() const final { return sources_.size(); } + const Schema &schema() const final { + return *schema_; + } + private: + std::shared_ptr schema_; std::vector> sources_; size_t numRows_ = 0; }; @@ -62,12 +72,10 @@ std::vector> makeFlexVectorsFromArrays( const std::vector> &arrays); } // namespace -//ImmerTableState::ImmerTableState(const ColumnDefinitions &colDefs) -// std::vector> flexVectors) : -// flexVectors_(std::move(flexVectors)) {} - -ImmerTableState::ImmerTableState(const ColumnDefinitions &colDefs) { - flexVectors_ = makeFlexVectorsFromColDefs(colDefs); +ImmerTableState::ImmerTableState(std::shared_ptr colDefs) : + colDefs_(std::move(colDefs)) { + schema_ = std::make_shared(colDefs_->vec()); + flexVectors_ = makeFlexVectorsFromColDefs(*colDefs_); } ImmerTableState::~ImmerTableState() = default; @@ -189,13 +197,13 @@ std::shared_ptr
ImmerTableState::snapshot() const { for (const auto &fv : flexVectors_) { columnSources.push_back(fv->makeColumnSource()); } - return std::make_shared(std::move(columnSources), spaceMapper_.size()); + return std::make_shared(schema_, std::move(columnSources), spaceMapper_.size()); } namespace { -MyTable::MyTable(std::vector> sources, size_t numRows) : - sources_(std::move(sources)), numRows_(numRows) {} +MyTable::MyTable(std::shared_ptr schema, std::vector> sources, + size_t numRows) : schema_(std::move(schema)), sources_(std::move(sources)), numRows_(numRows) {} MyTable::~MyTable() = default; std::shared_ptr MyTable::getRowSequence() const { @@ -208,7 +216,11 @@ std::shared_ptr MyTable::getRowSequence() const { struct FlexVectorMaker final { template void operator()() { - result_ = AbstractFlexVectorBase::create(immer::flex_vector()); + if constexpr(isNumericType()) { + result_ = std::make_unique>(); + } else { + result_ = std::make_unique>(); + } } std::unique_ptr result_; diff --git a/cpp-client/deephaven/client/src/subscription/shift_processor.cc b/cpp-client/deephaven/client/src/subscription/shift_processor.cc index a0943ac4dc4..77a9fd8c400 100644 --- a/cpp-client/deephaven/client/src/subscription/shift_processor.cc +++ b/cpp-client/deephaven/client/src/subscription/shift_processor.cc @@ -2,6 +2,7 @@ * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending */ #include "deephaven/client/subscription/shift_processor.h" +#include "deephaven/client/utility/utility.h" namespace deephaven::client::subscription { void ShiftProcessor::applyShiftData(const RowSequence &firstIndex, const RowSequence &lastIndex, @@ -24,9 +25,10 @@ void ShiftProcessor::applyShiftData(const RowSequence &firstIndex, const RowSequ }; { uint64_t first, last, dest; - while (startIter->tryGetNext(&first)) { - if (!endIter->tryGetNext(&last) || !destIter->tryGetNext(&dest)) { - throw std::runtime_error("Sequences not of same size"); + while (startIter.tryGetNext(&first)) { + if (!endIter.tryGetNext(&last) || !destIter.tryGetNext(&dest)) { + const char *message = "Sequences not of same size"; + throw std::runtime_error(DEEPHAVEN_DEBUG_MSG(message)); } if (dest >= first) { positiveShifts.emplace_back(first, last, dest); diff --git a/cpp-client/deephaven/client/src/subscription/subscribe_thread.cc b/cpp-client/deephaven/client/src/subscription/subscribe_thread.cc index e97c770fe9b..04814213e38 100644 --- a/cpp-client/deephaven/client/src/subscription/subscribe_thread.cc +++ b/cpp-client/deephaven/client/src/subscription/subscribe_thread.cc @@ -33,8 +33,7 @@ class SubscribeState final : public Callback<> { SubscribeState(std::shared_ptr server, std::vector ticketBytes, std::shared_ptr colDefs, std::promise> promise, - std::shared_ptr callback, - bool wantImmer); + std::shared_ptr callback); void invoke() final; private: @@ -45,7 +44,6 @@ class SubscribeState final : public Callback<> { std::shared_ptr colDefs_; std::promise> promise_; std::shared_ptr callback_; - bool wantImmer_ = false; }; // A simple extension to arrow::Buffer that owns its DetachedBuffer storage @@ -64,13 +62,12 @@ std::shared_ptr startSubscribeThread( Executor *flightExecutor, std::shared_ptr columnDefinitions, const Ticket &ticket, - std::shared_ptr callback, - bool wantImmer) { + std::shared_ptr callback) { std::promise> promise; auto future = promise.get_future(); std::vector ticketBytes(ticket.ticket().begin(), ticket.ticket().end()); auto ss = std::make_shared(std::move(server), std::move(ticketBytes), - std::move(columnDefinitions), std::move(promise), std::move(callback), wantImmer); + std::move(columnDefinitions), std::move(promise), std::move(callback)); flightExecutor->invoke(std::move(ss)); return future.get(); } @@ -78,9 +75,9 @@ std::shared_ptr startSubscribeThread( namespace { SubscribeState::SubscribeState(std::shared_ptr server, std::vector ticketBytes, std::shared_ptr colDefs, std::promise> promise, - std::shared_ptr callback, bool wantImmer) : + std::shared_ptr callback) : server_(std::move(server)), ticketBytes_(std::move(ticketBytes)), colDefs_(std::move(colDefs)), - promise_(std::move(promise)), callback_(std::move(callback)), wantImmer_(wantImmer) {} + promise_(std::move(promise)), callback_(std::move(callback)) {} void SubscribeState::invoke() { try { @@ -152,7 +149,7 @@ std::shared_ptr SubscribeState::invokeHelper() { // Run forever (until error or cancellation) auto processor = UpdateProcessor::startThread(std::move(fsr), std::move(colDefs_), - std::move(callback_), wantImmer_); + std::move(callback_)); return std::make_shared(std::move(processor)); } diff --git a/cpp-client/deephaven/client/src/subscription/update_processor.cc b/cpp-client/deephaven/client/src/subscription/update_processor.cc index 30005c264c2..98290d0fd3b 100644 --- a/cpp-client/deephaven/client/src/subscription/update_processor.cc +++ b/cpp-client/deephaven/client/src/subscription/update_processor.cc @@ -19,7 +19,6 @@ #include "deephaven/client/ticking.h" #include "deephaven/flatbuf/Barrage_generated.h" -using deephaven::client::ClassicTickingUpdate; using deephaven::client::chunk::ChunkFiller; using deephaven::client::chunk::ChunkMaker; using deephaven::client::chunk::UInt64Chunk; @@ -74,144 +73,130 @@ std::optional extractMetadata( std::shared_ptr UpdateProcessor::startThread( std::unique_ptr fsr, std::shared_ptr colDefs, - std::shared_ptr callback, - bool wantImmer) { + std::shared_ptr callback) { auto result = std::make_shared(std::move(fsr), std::move(colDefs), std::move(callback)); - std::thread t(&UpdateProcessor::runForever, result, wantImmer); + std::thread t(&UpdateProcessor::runForever, result); t.detach(); return result; } UpdateProcessor::UpdateProcessor(std::unique_ptr fsr, std::shared_ptr colDefs, std::shared_ptr callback) : - fsr_(std::move(fsr)), colDefs_(std::move(colDefs)), callback_(std::move(callback)) {} + fsr_(std::move(fsr)), colDefs_(std::move(colDefs)), callback_(std::move(callback)), + cancelled_(false) {} UpdateProcessor::~UpdateProcessor() = default; void UpdateProcessor::cancel() { + cancelled_ = true; fsr_->Cancel(); } -void UpdateProcessor::runForever(const std::shared_ptr &self, bool wantImmer) { +void UpdateProcessor::runForever(const std::shared_ptr &self) { std::cerr << "UpdateProcessor is starting.\n"; std::exception_ptr eptr; try { - if (wantImmer) { - self->immerRunForeverHelper(); - } else { - self->classicRunForeverHelper(); - } + self->runForeverHelper(); } catch (...) { - eptr = std::current_exception(); - self->callback_->onFailure(eptr); + // If the thread was been cancelled via explicit user action, then swallow all errors. + if (!self->cancelled_) { + eptr = std::current_exception(); + self->callback_->onFailure(eptr); + } } - std::cerr << "UpdateProcessor is exiting.\n"; } -void UpdateProcessor::classicRunForeverHelper() { - ClassicTableState state(*colDefs_); - - // In this loop we process Arrow Flight messages until error or cancellation. - arrow::flight::FlightStreamChunk flightStreamChunk; - while (true) { - okOrThrow(DEEPHAVEN_EXPR_MSG(fsr_->Next(&flightStreamChunk))); +std::pair, std::shared_ptr
> processRemoves( + const std::shared_ptr
&beforeRemoves, ImmerTableState *state, + const ExtractedMetadata &md) { + if (md.removedRows_->empty()) { + auto empty = RowSequence::createEmpty(); + auto afterRemoves = beforeRemoves; + return {std::move(empty), std::move(afterRemoves)}; + } - // Parse all the metadata out of the Barrage message before we advance the cursor past it. - auto mdo = extractMetadata(flightStreamChunk); - if (!mdo.has_value()) { - continue; - } - auto &md = *mdo; + auto removedRowsIndexSpace = state->erase(*md.removedRows_); + auto afterRemoves = state->snapshot(); + return {std::move(removedRowsIndexSpace), std::move(afterRemoves)}; +} - // Correct order to process all this info is: - // 1. removes - // 2. shifts - // 3. adds - // 4. modifies +std::pair, std::shared_ptr
> processAdds( + const std::shared_ptr
&beforeAdds, ImmerTableState *state, + const ExtractedMetadata &md, size_t numCols, + arrow::flight::FlightStreamReader *fsr, arrow::flight::FlightStreamChunk *flightStreamChunk) { + if (md.numAdds_ == 0) { + auto empty = RowSequence::createEmpty(); + auto afterAdds = beforeAdds; + return {std::move(empty), std::move(afterAdds)}; + } - // 1. Removes - auto removedRowsKeySpace = std::move(md.removedRows_); - auto removedRowsIndexSpace = state.erase(*removedRowsKeySpace); + auto addedRowsIndexSpace = state->addKeys(*md.addedRows_); - // 2. Shifts - state.applyShifts(*md.shiftStartIndex_, *md.shiftEndIndex_, *md.shiftDestIndex_); + // Copy everything. + auto rowsRemaining = addedRowsIndexSpace->take(addedRowsIndexSpace->size()); - // 3. Adds - auto addedRowsKeySpace = RowSequence::createEmpty(); - auto addedRowsIndexSpace = UInt64Chunk::create(0); - if (md.numAdds_ != 0) { - addedRowsKeySpace = std::move(md.addedRows_); - addedRowsIndexSpace = state.addKeys(*addedRowsKeySpace); - - // Copy everything. - auto rowsRemaining = addedRowsIndexSpace.take(addedRowsIndexSpace.size()); - - auto processAddBatch = [&state, &rowsRemaining]( - const std::vector> &data) { - if (data.empty()) { - return; - } - auto size = data[0]->length(); - auto rowsToAddThisTime = rowsRemaining.take(size); - rowsRemaining = rowsRemaining.drop(size); - state.addData(data, rowsToAddThisTime); - }; - BatchParser::parseBatches(*colDefs_, md.numAdds_, false, fsr_.get(), &flightStreamChunk, - processAddBatch); - - if (md.numMods_ != 0) { - // Currently the FlightStreamReader is pointing to the last add record. We need to advance - // it so it points to the first mod record. - okOrThrow(DEEPHAVEN_EXPR_MSG(fsr_->Next(&flightStreamChunk))); - } + auto processAddBatch = [state, &rowsRemaining]( + const std::vector> &data) { + if (data.empty()) { + return; } + auto size = data[0]->length(); + auto rowsToAddThisTime = rowsRemaining->take(size); + rowsRemaining = rowsRemaining->drop(size); + state->addData(data, *rowsToAddThisTime); + }; + BatchParser::parseBatches(numCols, md.numAdds_, false, fsr, flightStreamChunk, + processAddBatch); + + if (md.numMods_ != 0) { + // Currently the FlightStreamReader is pointing to the last add record. We need to advance + // it so it points to the first mod record. + okOrThrow(DEEPHAVEN_EXPR_MSG(fsr->Next(flightStreamChunk))); + } + auto afterAdds = state->snapshot(); + return {std::move(addedRowsIndexSpace), std::move(afterAdds)}; +} - auto ncols = colDefs_->vec().size(); +std::pair>, std::shared_ptr
> processModifies( + const std::shared_ptr
&beforeModifies, ImmerTableState *state, + const ExtractedMetadata &md, size_t numCols, arrow::flight::FlightStreamReader *fsr, + arrow::flight::FlightStreamChunk *flightStreamChunk) { + if (md.numMods_ == 0) { + auto afterModifies = beforeModifies; + return {{}, std::move(afterModifies)}; + } + auto modifiedRowsIndexSpace = state->modifyKeys(md.modifiedRows_); + // Local copy of modifiedRowsIndexSpace + auto keysRemaining = makeReservedVector>(numCols); + for (const auto &keys: modifiedRowsIndexSpace) { + keysRemaining.push_back(keys->take(keys->size())); + } - // 4. Modifies - auto modifiedRowsKeySpace = std::move(md.modifiedRows_); - auto modifiedRowsIndexSpace = state.modifyKeys(modifiedRowsKeySpace); - if (md.numMods_ != 0) { - // Local copy of modifiedRowsIndexSpace - auto keysRemaining = makeReservedVector(ncols); - for (const auto &keys : modifiedRowsIndexSpace) { - keysRemaining.push_back(keys.take(keys.size())); - } - - std::vector keysToModifyThisTime(ncols); - - auto processModifyBatch = [&state, &keysRemaining, &keysToModifyThisTime, ncols]( - const std::vector> &data) { - if (data.size() != ncols) { - throw std::runtime_error(stringf("data.size() != ncols (%o != %o)", data.size(), ncols)); - } - for (size_t i = 0; i < data.size(); ++i) { - const auto &src = data[i]; - auto &krm = keysRemaining[i]; - keysToModifyThisTime[i] = krm.take(src->length()); - krm = krm.drop(src->length()); - } - state.modifyData(data, keysToModifyThisTime); - }; - - BatchParser::parseBatches(*colDefs_, md.numMods_, true, fsr_.get(), &flightStreamChunk, - processModifyBatch); - } + std::vector> keysToModifyThisTime(numCols); - auto currentTableKeySpace = state.snapshot(); - auto currentTableIndexSpace = state.snapshotUnwrapped(); + auto processModifyBatch = [state, &keysRemaining, &keysToModifyThisTime, numCols]( + const std::vector> &data) { + if (data.size() != numCols) { + auto message = stringf("Expected % cols, got %o", numCols, data.size()); + throw std::runtime_error(DEEPHAVEN_DEBUG_MSG(message)); + } + for (size_t i = 0; i < data.size(); ++i) { + const auto &src = data[i]; + auto &krm = keysRemaining[i]; + keysToModifyThisTime[i] = krm->take(src->length()); + krm = krm->drop(src->length()); + } + state->modifyData(data, keysToModifyThisTime); + }; - ClassicTickingUpdate update(std::move(removedRowsKeySpace), std::move(removedRowsIndexSpace), - std::move(addedRowsKeySpace), std::move(addedRowsIndexSpace), - std::move(modifiedRowsKeySpace), std::move(modifiedRowsIndexSpace), - std::move(currentTableKeySpace), std::move(currentTableIndexSpace)); - callback_->onTick(update); - } + BatchParser::parseBatches(numCols, md.numMods_, true, fsr, flightStreamChunk, processModifyBatch); + auto afterModifies = state->snapshot(); + return {std::move(modifiedRowsIndexSpace), std::move(afterModifies)}; } -void UpdateProcessor::immerRunForeverHelper() { - ImmerTableState state(*colDefs_); +void UpdateProcessor::runForeverHelper() { + ImmerTableState state(colDefs_); // In this loop we process Arrow Flight messages until error or cancellation. arrow::flight::FlightStreamChunk flightStreamChunk; @@ -231,82 +216,27 @@ void UpdateProcessor::immerRunForeverHelper() { // 3. adds // 4. modifies + auto prev = state.snapshot(); + // 1. Removes - auto beforeRemoves = state.snapshot(); - auto removedRowsKeySpace = std::move(md.removedRows_); - auto removedRowsIndexSpace = state.erase(*removedRowsKeySpace); + auto [removes, afterRemoves] = processRemoves(prev, &state, md); // 2. Shifts state.applyShifts(*md.shiftStartIndex_, *md.shiftEndIndex_, *md.shiftDestIndex_); // 3. Adds - auto addedRowsKeySpace = RowSequence::createEmpty(); - auto addedRowsIndexSpace = RowSequence::createEmpty(); - if (md.numAdds_ != 0) { - addedRowsKeySpace = std::move(md.addedRows_); - addedRowsIndexSpace = state.addKeys(*addedRowsKeySpace); - - // Copy everything. - auto rowsRemaining = addedRowsIndexSpace->take(addedRowsIndexSpace->size()); - - auto processAddBatch = [&state, &rowsRemaining]( - const std::vector> &data) { - if (data.empty()) { - return; - } - auto size = data[0]->length(); - auto rowsToAddThisTime = rowsRemaining->take(size); - rowsRemaining = rowsRemaining->drop(size); - state.addData(data, *rowsToAddThisTime); - }; - BatchParser::parseBatches(*colDefs_, md.numAdds_, false, fsr_.get(), &flightStreamChunk, - processAddBatch); - - if (md.numMods_ != 0) { - // Currently the FlightStreamReader is pointing to the last add record. We need to advance - // it so it points to the first mod record. - okOrThrow(DEEPHAVEN_EXPR_MSG(fsr_->Next(&flightStreamChunk))); - } - } - - auto beforeModifies = state.snapshot(); - - auto ncols = colDefs_->vec().size(); + auto [adds, afterAdds] = processAdds(afterRemoves, &state, md, colDefs_->size(), fsr_.get(), + &flightStreamChunk); // 4. Modifies - auto modifiedRowsKeySpace = std::move(md.modifiedRows_); - auto modifiedRowsIndexSpace = state.modifyKeys(modifiedRowsKeySpace); - if (md.numMods_ != 0) { - // Local copy of modifiedRowsIndexSpace - auto keysRemaining = makeReservedVector>(ncols); - for (const auto &keys : modifiedRowsIndexSpace) { - keysRemaining.push_back(keys->take(keys->size())); - } - - std::vector> keysToModifyThisTime(ncols); - - auto processModifyBatch = [&state, &keysRemaining, &keysToModifyThisTime, ncols]( - const std::vector> &data) { - if (data.size() != ncols) { - throw std::runtime_error(stringf("data.size() != ncols (%o != %o)", data.size(), ncols)); - } - for (size_t i = 0; i < data.size(); ++i) { - const auto &src = data[i]; - auto &krm = keysRemaining[i]; - keysToModifyThisTime[i] = krm->take(src->length()); - krm = krm->drop(src->length()); - } - state.modifyData(data, keysToModifyThisTime); - }; - - BatchParser::parseBatches(*colDefs_, md.numMods_, true, fsr_.get(), &flightStreamChunk, - processModifyBatch); - } - - auto current = state.snapshot(); - ImmerTickingUpdate update(std::move(beforeRemoves), std::move(beforeModifies), - std::move(current), std::move(removedRowsIndexSpace), std::move(modifiedRowsIndexSpace), - std::move(addedRowsIndexSpace)); + auto [modifies, afterModifies] = processModifies(afterAdds, &state, md, colDefs_->size(), + fsr_.get(), &flightStreamChunk); + + // These are convenience copies of the user which might + TickingUpdate update(std::move(prev), + std::move(removes), std::move(afterRemoves), + std::move(adds), std::move(afterAdds), + std::move(modifies), std::move(afterModifies)); callback_->onTick(std::move(update)); } } diff --git a/cpp-client/deephaven/client/src/table/table.cc b/cpp-client/deephaven/client/src/table/table.cc index d75fcbabf4c..d37228f69f7 100644 --- a/cpp-client/deephaven/client/src/table/table.cc +++ b/cpp-client/deephaven/client/src/table/table.cc @@ -3,5 +3,365 @@ */ #include "deephaven/client/table/table.h" +#include "deephaven/client/chunk/chunk_maker.h" +#include "deephaven/client/chunk/chunk.h" +#include "deephaven/client/container/row_sequence.h" +#include "deephaven/client/utility/utility.h" + +using deephaven::client::chunk::AnyChunk; +using deephaven::client::chunk::BooleanChunk; +using deephaven::client::chunk::ChunkMaker; +using deephaven::client::column::ColumnSource; +using deephaven::client::container::RowSequence; +using deephaven::client::container::RowSequenceIterator; +using deephaven::client::utility::makeReservedVector; +using deephaven::client::utility::separatedList; +using deephaven::client::utility::stringf; + namespace deephaven::client::table { +namespace { +void printTableData(std::ostream &stream, const Table &table, + const std::vector &whichCols, + const std::vector> &rowSequences, + bool wantHeaders, bool wantRowNumbers, bool highlightCells); +} // namespace + +std::shared_ptr Table::getColumn(std::string_view name, bool strict) const { + // TODO(kosak): improve linear search. + const auto &cols = schema().columns(); + for (size_t i = 0; i < cols.size(); ++i) { + if (cols[i].first == name) { + return getColumn(i); + } + } + // Not found: check strictness flag. + if (strict) { + auto message = stringf("Column name '%o' not found", name); + throw std::runtime_error(DEEPHAVEN_DEBUG_MSG(message)); + } + return {}; +} + +internal::TableStreamAdaptor Table::stream(bool wantHeaders, bool wantRowNumbers) const { + std::vector> rowSequences{getRowSequence()}; + return {*this, std::move(rowSequences), wantHeaders, wantRowNumbers, false}; +} + +internal::TableStreamAdaptor Table::stream(bool wantHeaders, bool wantRowNumbers, + std::shared_ptr rowSequence) const { + std::vector> rowSequences{std::move(rowSequence)}; + return {*this, std::move(rowSequences), wantHeaders, wantRowNumbers, false}; +} + +internal::TableStreamAdaptor Table::stream(bool wantHeaders, bool wantRowNumbers, + std::vector> rowSequences) const { + return {*this, std::move(rowSequences), wantHeaders, wantRowNumbers, true}; +} + +Schema::Schema(std::vector>> columns) : + columns_(std::move(columns)) {} +Schema::Schema(Schema &&other) noexcept = default; +Schema &Schema::operator=(Schema &&other) noexcept = default; +Schema::~Schema() = default; + +namespace internal { +std::ostream &operator<<(std::ostream &s, const TableStreamAdaptor &o) { + const auto &t = o.table_; + auto numCols = t.numColumns(); + auto whichCols = makeReservedVector(numCols); + for (size_t i = 0; i < numCols; ++i) { + whichCols.push_back(i); + } + printTableData(s, t, whichCols, o.rowSequences_, o.wantHeaders_, o.wantRowNumbers_, o.highlightCells_); + return s; +} +} // namespace internal + +namespace { +class ArrayRowSequence final : public RowSequence { +public: + static std::shared_ptr create(std::shared_ptr data, + const uint64_t *begin, const uint64_t *end); + + ArrayRowSequence(std::shared_ptr data, const uint64_t *begin, const uint64_t *end); + ~ArrayRowSequence() final; + + std::shared_ptr take(size_t size) const final; + std::shared_ptr drop(size_t size) const final; + void forEachChunk(const std::function &f) const final; + + size_t size() const final { + return end_ - begin_; + } + +private: + std::shared_ptr data_; + const uint64_t *begin_ = nullptr; + const uint64_t *end_ = nullptr; +}; + +class ElementStreamer final { +public: + ElementStreamer(std::ostream &s, size_t index, bool nullFlag, bool highlight) : + s_(s), index_(index), nullFlag_(nullFlag), highlight_(highlight) {} + + template + void operator()(const T &chunk) const { + if (highlight_) { + s_ << '*'; + } + + if (nullFlag_) { + s_ << "null"; + } else { + render(chunk.data()[index_]); + } + + if (highlight_) { + s_ << '*'; + } + } + +private: + template + void render(const T &item) const { + s_ << item; + } + + void render(const bool &item) const { + s_ << (item ? "true" : "false"); + } + + std::ostream &s_; + size_t index_ = 0; + bool nullFlag_ = false; + bool highlight_ = false; +}; + +struct RowSequenceState { + explicit RowSequenceState(RowSequenceIterator iterator, size_t chunkSize); + RowSequenceState(RowSequenceState &&other) noexcept; + ~RowSequenceState(); + + RowSequenceIterator iterator_; + std::optional currentValue_; + std::unique_ptr isPresent_; +}; + +class RowMerger { +public: + RowMerger(std::vector iterators, size_t chunkSize); + ~RowMerger(); + + std::shared_ptr getNextChunk(); + + bool isCellPresent(size_t colIndex, size_t chunkOffset) const; + +private: + size_t chunkSize_ = 0; + std::vector rowSequenceStates_; + /** + * This is a shared ponter because we share it with the ArrayRowSequence that we return + * from getNextChunk. size = chunkSize_ + */ + std::shared_ptr build_; +}; + +void printTableData(std::ostream &stream, const Table &table, + const std::vector &whichCols, + const std::vector> &rowSequences, + bool wantHeaders, bool wantRowNumbers, bool highlightCells) { + if (wantHeaders) { + const char *separator = ""; + if (wantRowNumbers) { + stream << "[Row]"; + separator = "\t"; + } + for (auto colIndex : whichCols) { + stream << separator << table.schema().columns()[colIndex].first; + separator = "\t"; + } + stream << std::endl; + } + + if (whichCols.empty() || rowSequences.empty()) { + return; + } + + const size_t chunkSize = 8192; + + auto numCols = whichCols.size(); + auto dataChunks = makeReservedVector(numCols); + auto nullFlagChunks = makeReservedVector(numCols); + for (size_t i = 0; i < numCols; ++i) { + const auto &c = table.getColumn(i); + auto dataChunk = ChunkMaker::createChunkFor(*c, chunkSize); + auto nullFlagChunk = BooleanChunk::create(chunkSize); + dataChunks.push_back(std::move(dataChunk)); + nullFlagChunks.push_back(std::move(nullFlagChunk)); + } + + auto iterators = makeReservedVector(rowSequences.size()); + for (const auto &rs : rowSequences) { + iterators.push_back(rs->getRowSequenceIterator()); + } + + RowMerger merger(std::move(iterators), chunkSize); + + while (true) { + auto chunkOfRows = merger.getNextChunk(); + auto thisSize = chunkOfRows->size(); + if (thisSize == 0) { + break; + } + + for (size_t i = 0; i < numCols; ++i) { + const auto colNum = whichCols[i]; + const auto &c = table.getColumn(colNum); + auto &dataChunk = dataChunks[colNum].unwrap(); + auto &nullFlagChunk = nullFlagChunks[colNum]; + c->fillChunk(*chunkOfRows, &dataChunk, &nullFlagChunk); + } + + // To print out the optional row number + auto rowsIter = chunkOfRows->getRowSequenceIterator(); + + for (size_t chunkOffset = 0; chunkOffset < thisSize; ++chunkOffset) { + const char *separator = ""; + if (wantRowNumbers) { + uint64_t rowNum; + if (!rowsIter.tryGetNext(&rowNum)) { + throw std::runtime_error(DEEPHAVEN_DEBUG_MSG("Impossible: no more rows")); + } + stream << '[' << rowNum << "] "; + separator = "\t"; + } + + for (size_t i = 0; i < numCols; ++i) { + stream << separator; + separator = "\t"; + auto nullFlag = nullFlagChunks[i].data()[chunkOffset]; + auto highlight = highlightCells && merger.isCellPresent(i, chunkOffset); + ElementStreamer es(stream, chunkOffset, nullFlag, highlight); + dataChunks[i].visit(es); + } + + stream << std::endl; + } + } +} + +std::shared_ptr +ArrayRowSequence::create(std::shared_ptr data, const uint64_t *begin, + const uint64_t *end) { + return std::make_shared(std::move(data), begin, end); +} + +ArrayRowSequence::ArrayRowSequence(std::shared_ptr data, const uint64_t *begin, + const uint64_t *end) : data_(std::move(data)), begin_(begin), end_(end) {} + +ArrayRowSequence::~ArrayRowSequence() = default; + +std::shared_ptr ArrayRowSequence::take(size_t size) const { + auto sizeToUse = std::min(size, this->size()); + return create(data_, begin_, begin_ + sizeToUse); +} + +std::shared_ptr ArrayRowSequence::drop(size_t size) const { + auto sizeToUse = std::min(size, this->size()); + return create(data_, begin_ + sizeToUse, end_); +} + +void ArrayRowSequence::forEachChunk(const std::function &f) const { + const auto *rangeStart = begin_; + while (rangeStart != end_) { + auto beginKey = *rangeStart; + const auto *rangeEnd = rangeStart + 1; + auto endKey = beginKey + 1; + while (rangeEnd != end_ && *rangeEnd == endKey) { + ++rangeEnd; + ++endKey; + } + f(beginKey, endKey); + rangeStart = rangeEnd; + } +} + +RowSequenceState::RowSequenceState(RowSequenceIterator iterator, size_t chunkSize) : + iterator_(std::move(iterator)), isPresent_(std::make_unique(chunkSize)) { + uint64_t value; + if (iterator_.tryGetNext(&value)) { + currentValue_ = value; + } +} +RowSequenceState::RowSequenceState(RowSequenceState &&other) noexcept = default; +RowSequenceState::~RowSequenceState() = default; + +RowMerger::RowMerger(std::vector iterators, size_t chunkSize) : + chunkSize_(chunkSize) { + + rowSequenceStates_ = makeReservedVector(iterators.size()); + for (auto &iter : iterators) { + rowSequenceStates_.emplace_back(std::move(iter), chunkSize); + } + build_ = std::shared_ptr(new uint64_t[chunkSize]); +} + +RowMerger::~RowMerger() = default; + +std::shared_ptr RowMerger::getNextChunk() { + size_t destIndex; + uint64_t *buildp = build_.get(); + for (destIndex = 0; destIndex < chunkSize_; ++destIndex) { + // Simplistic priority queue. If performance becomes an issue, this should be rewritten as a + // legit priority queue. + + // First calculate the minimum value among the current values (if one exists) + std::optional minValue; + for (const auto &rss : rowSequenceStates_) { + const auto &cv = rss.currentValue_; + if (!cv.has_value()) { + continue; + } + if (!minValue.has_value() || *cv < *minValue) { + minValue = *cv; + } + } + + // If no values found, we are done. + if (!minValue.has_value()) { + break; + } + + // Store the minimum value, calculate the isPresent flag, and advance the iterators that match + // the minimum value. + buildp[destIndex] = *minValue; + + // Advance the iterators that match the minimum value. + for (auto &rss : rowSequenceStates_) { + auto &cv = rss.currentValue_; + if (!cv.has_value() || *cv != *minValue) { + rss.isPresent_[destIndex] = false; + continue; + } + rss.isPresent_[destIndex] = true; + + // Bump to next if you can + uint64_t value; + if (rss.iterator_.tryGetNext(&value)) { + cv = value; + } else { + cv.reset(); + } + } + } + + return ArrayRowSequence::create(build_, build_.get(), build_.get() + destIndex); +} + +bool RowMerger::isCellPresent(size_t colIndex, size_t chunkOffset) const { + auto colIndexToUse = colIndex < rowSequenceStates_.size() ? colIndex : 0; + return rowSequenceStates_[colIndexToUse].isPresent_[chunkOffset]; +} +} // namespace } // namespace deephaven::client::table diff --git a/cpp-client/deephaven/client/src/ticking.cc b/cpp-client/deephaven/client/src/ticking.cc index cd3bf250e3f..c9e45aa9041 100644 --- a/cpp-client/deephaven/client/src/ticking.cc +++ b/cpp-client/deephaven/client/src/ticking.cc @@ -4,40 +4,14 @@ #include "deephaven/client/ticking.h" namespace deephaven::client { -ClassicTickingUpdate::ClassicTickingUpdate(std::shared_ptr removedRowsKeySpace, - UInt64Chunk removedRowsIndexSpace, - std::shared_ptr addedRowsKeySpace, - UInt64Chunk addedRowsIndexSpace, - std::vector> modifiedRowsKeySpace, - std::vector modifiedRowsIndexSpace, - std::shared_ptr
currentTableKeySpace, - std::shared_ptr
currentTableIndexSpace) : - removedRowsKeySpace_(std::move(removedRowsKeySpace)), - removedRowsIndexSpace_(std::move(removedRowsIndexSpace)), - addedRowsKeySpace_(std::move(addedRowsKeySpace)), - addedRowsIndexSpace_(std::move(addedRowsIndexSpace)), - modifiedRowsKeySpace_(std::move(modifiedRowsKeySpace)), - modifiedRowsIndexSpace_(std::move(modifiedRowsIndexSpace)), - currentTableKeySpace_(std::move(currentTableKeySpace)), - currentTableIndexSpace_(std::move(currentTableIndexSpace)) {} -ClassicTickingUpdate::ClassicTickingUpdate(ClassicTickingUpdate &&other) noexcept = default; -ClassicTickingUpdate &ClassicTickingUpdate::operator=(ClassicTickingUpdate &&other) noexcept = default; -ClassicTickingUpdate::~ClassicTickingUpdate() = default; - -ImmerTickingUpdate::ImmerTickingUpdate(std::shared_ptr
beforeRemoves, - std::shared_ptr
beforeModifies, - std::shared_ptr
current, - std::shared_ptr removed, - std::vector> modified, - std::shared_ptr added) : beforeRemoves_(std::move(beforeRemoves)), - beforeModifies_(std::move(beforeModifies)), - current_(std::move(current)), - removed_(std::move(removed)), - modified_(std::move(modified)), - added_(std::move(added)) {} - -ImmerTickingUpdate::ImmerTickingUpdate(ImmerTickingUpdate &&other) noexcept = default; -ImmerTickingUpdate &ImmerTickingUpdate::operator=(ImmerTickingUpdate &&other) noexcept = default; -ImmerTickingUpdate::~ImmerTickingUpdate() = default; - +TickingUpdate::TickingUpdate(std::shared_ptr
prev, std::shared_ptr removedRows, + std::shared_ptr
afterRemoves, std::shared_ptr addedRows, + std::shared_ptr
afterAdds, std::vector> modifiedRows, + std::shared_ptr
afterModifies) : prev_(std::move(prev)), + removedRows_(std::move(removedRows)), afterRemoves_(std::move(afterRemoves)), + addedRows_(std::move(addedRows)), afterAdds_(std::move(afterAdds)), + modifiedRows_(std::move(modifiedRows)), afterModifies_(std::move(afterModifies)) {} +TickingUpdate::TickingUpdate(TickingUpdate &&other) noexcept = default; +TickingUpdate &TickingUpdate::operator=(TickingUpdate &&other) noexcept = default; +TickingUpdate::~TickingUpdate() = default; } // namespace deephaven::client diff --git a/cpp-client/deephaven/client/src/types.cc b/cpp-client/deephaven/client/src/types.cc index fafe8f3c64b..28724d9e05f 100644 --- a/cpp-client/deephaven/client/src/types.cc +++ b/cpp-client/deephaven/client/src/types.cc @@ -65,7 +65,7 @@ void DateTime::streamIrisRepresentation(std::ostream &s) const { size_t oneBillion = 1000000000; time_t timeSecs = nanos_ / oneBillion; auto nanos = nanos_ % oneBillion; - struct tm tm; + struct tm tm = {}; gmtime_r(&timeSecs, &tm); char dateBuffer[32]; // ample char nanosBuffer[32]; // ample diff --git a/cpp-client/deephaven/client/src/utility/utility.cc b/cpp-client/deephaven/client/src/utility/utility.cc index 868c4fef0cb..d1f0f13a205 100644 --- a/cpp-client/deephaven/client/src/utility/utility.cc +++ b/cpp-client/deephaven/client/src/utility/utility.cc @@ -110,6 +110,13 @@ std::ostream &operator<<(std::ostream &s, const DebugInfo &o) { return streamf(s, "%o@%o:%o args=(%o))", o.func_, o.file_, o.line_, o.args_); } +namespace internal { +void trueOrThrowHelper(const DebugInfo &debugInfo) { + auto message = stringf("Assertion failed: %o", debugInfo); + throw std::runtime_error(message); +} +} // namespace internal + void okOrThrow(const DebugInfo &debugInfo, const arrow::Status &status) { if (status.ok()) { return; diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/LICENSE.txt b/cpp-client/deephaven/client/third_party/flatbuffers/LICENSE.txt deleted file mode 100644 index 854c7efc32f..00000000000 --- a/cpp-client/deephaven/client/third_party/flatbuffers/LICENSE.txt +++ /dev/null @@ -1,7 +0,0 @@ -TODO: put correct license here -indicating we are using Flatbuffers v2.0.6 -modifications are: -- copied only the files we needed -- moved the namespace of the code to a Deephaven-internal namespace, - so as not to conflict with any other flatbuffers being linked - (such as the one inside Arrow) diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/README.md b/cpp-client/deephaven/client/third_party/flatbuffers/README.md new file mode 100644 index 00000000000..7cd23f293e3 --- /dev/null +++ b/cpp-client/deephaven/client/third_party/flatbuffers/README.md @@ -0,0 +1,13 @@ +This directory contains a vendored version of Flatbuffers v2.0.6, +with the following changes: + +1. We have copied only the files we need: + allocator.h array.h base.h buffer.h buffer_ref.h default_allocator.h + detached_buffer.h flatbuffer_builder.h flatbuffers.h stl_emulation.h + string.h struct.h table.h util.h vector_downward.h vector.h verifier.h +2. We have moved the namespace of the code to a Deephaven-internal namespace, + so as not to conflict with any other flatbuffers being indirectly linked + from some other library (such as the one inside Arrow). +3. The patch representing step 2 is in the file patch.001 in this directory. + + diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/allocator.h b/cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/allocator.h similarity index 100% rename from cpp-client/deephaven/client/third_party/flatbuffers/allocator.h rename to cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/allocator.h diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/array.h b/cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/array.h similarity index 100% rename from cpp-client/deephaven/client/third_party/flatbuffers/array.h rename to cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/array.h diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/base.h b/cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/base.h similarity index 100% rename from cpp-client/deephaven/client/third_party/flatbuffers/base.h rename to cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/base.h diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/buffer.h b/cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/buffer.h similarity index 100% rename from cpp-client/deephaven/client/third_party/flatbuffers/buffer.h rename to cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/buffer.h diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/buffer_ref.h b/cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/buffer_ref.h similarity index 100% rename from cpp-client/deephaven/client/third_party/flatbuffers/buffer_ref.h rename to cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/buffer_ref.h diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/default_allocator.h b/cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/default_allocator.h similarity index 100% rename from cpp-client/deephaven/client/third_party/flatbuffers/default_allocator.h rename to cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/default_allocator.h diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/detached_buffer.h b/cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/detached_buffer.h similarity index 100% rename from cpp-client/deephaven/client/third_party/flatbuffers/detached_buffer.h rename to cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/detached_buffer.h diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/flatbuffer_builder.h b/cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/flatbuffer_builder.h similarity index 100% rename from cpp-client/deephaven/client/third_party/flatbuffers/flatbuffer_builder.h rename to cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/flatbuffer_builder.h diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/flatbuffers.h b/cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/flatbuffers.h similarity index 100% rename from cpp-client/deephaven/client/third_party/flatbuffers/flatbuffers.h rename to cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/flatbuffers.h diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/stl_emulation.h b/cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/stl_emulation.h similarity index 100% rename from cpp-client/deephaven/client/third_party/flatbuffers/stl_emulation.h rename to cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/stl_emulation.h diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/string.h b/cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/string.h similarity index 100% rename from cpp-client/deephaven/client/third_party/flatbuffers/string.h rename to cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/string.h diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/struct.h b/cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/struct.h similarity index 100% rename from cpp-client/deephaven/client/third_party/flatbuffers/struct.h rename to cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/struct.h diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/table.h b/cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/table.h similarity index 100% rename from cpp-client/deephaven/client/third_party/flatbuffers/table.h rename to cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/table.h diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/util.h b/cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/util.h similarity index 100% rename from cpp-client/deephaven/client/third_party/flatbuffers/util.h rename to cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/util.h diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/vector.h b/cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/vector.h similarity index 100% rename from cpp-client/deephaven/client/third_party/flatbuffers/vector.h rename to cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/vector.h diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/vector_downward.h b/cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/vector_downward.h similarity index 100% rename from cpp-client/deephaven/client/third_party/flatbuffers/vector_downward.h rename to cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/vector_downward.h diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/verifier.h b/cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/verifier.h similarity index 100% rename from cpp-client/deephaven/client/third_party/flatbuffers/verifier.h rename to cpp-client/deephaven/client/third_party/flatbuffers/include/flatbuffers/verifier.h diff --git a/cpp-client/deephaven/client/third_party/flatbuffers/patch.001 b/cpp-client/deephaven/client/third_party/flatbuffers/patch.001 new file mode 100644 index 00000000000..11f934d027f --- /dev/null +++ b/cpp-client/deephaven/client/third_party/flatbuffers/patch.001 @@ -0,0 +1,350 @@ +diff -ur orig/allocator.h flatbuffers/allocator.h +--- orig/allocator.h 2022-09-07 19:04:45.122547635 -0400 ++++ flatbuffers/allocator.h 2022-08-06 02:58:02.748814653 -0400 +@@ -19,7 +19,12 @@ + + #include "flatbuffers/base.h" + +-namespace flatbuffers { ++// Move the vendored copy of flatbuffers to a private namespace so it doesn't conflict ++// with the flatbuffers namespace compiled into Arrow. ++namespace deephaven::third_party::flatbuffers {} ++namespace flatbuffers = deephaven::third_party::flatbuffers; ++ ++namespace deephaven::third_party::flatbuffers { + + // Allocator interface. This is flatbuffers-specific and meant only for + // `vector_downward` usage. +@@ -65,4 +70,4 @@ + + } // namespace flatbuffers + +-#endif // FLATBUFFERS_ALLOCATOR_H_ +\ No newline at end of file ++#endif // FLATBUFFERS_ALLOCATOR_H_ +diff -ur orig/array.h flatbuffers/array.h +--- orig/array.h 2022-09-07 19:04:45.122547635 -0400 ++++ flatbuffers/array.h 2022-08-06 02:58:02.748814653 -0400 +@@ -21,7 +21,12 @@ + #include "flatbuffers/stl_emulation.h" + #include "flatbuffers/vector.h" + +-namespace flatbuffers { ++// Move the vendored copy of flatbuffers to a private namespace so it doesn't conflict ++// with the flatbuffers namespace compiled into Arrow. ++namespace deephaven::third_party::flatbuffers {} ++namespace flatbuffers = deephaven::third_party::flatbuffers; ++ ++namespace deephaven::third_party::flatbuffers { + + // This is used as a helper type for accessing arrays. + template class Array { +diff -ur orig/base.h flatbuffers/base.h +--- orig/base.h 2022-09-07 19:04:45.122547635 -0400 ++++ flatbuffers/base.h 2022-08-06 02:58:02.748814653 -0400 +@@ -143,7 +143,11 @@ + #define FLATBUFFERS_VERSION_REVISION 6 + #define FLATBUFFERS_STRING_EXPAND(X) #X + #define FLATBUFFERS_STRING(X) FLATBUFFERS_STRING_EXPAND(X) +-namespace flatbuffers { ++// Move the vendored copy of flatbuffers to a private namespace so it doesn't conflict ++// with the flatbuffers namespace compiled into Arrow. ++namespace deephaven::third_party::flatbuffers {} ++namespace flatbuffers = deephaven::third_party::flatbuffers; ++namespace deephaven::third_party::flatbuffers { + // Returns version as string "MAJOR.MINOR.REVISION". + const char* FLATBUFFERS_VERSION(); + } +@@ -221,21 +225,21 @@ + // Check for std::string_view (in c++17) + #if __has_include() && (__cplusplus >= 201606 || (defined(_HAS_CXX17) && _HAS_CXX17)) + #include +- namespace flatbuffers { ++ namespace deephaven::third_party::flatbuffers { + typedef std::string_view string_view; + } + #define FLATBUFFERS_HAS_STRING_VIEW 1 + // Check for std::experimental::string_view (in c++14, compiler-dependent) + #elif __has_include() && (__cplusplus >= 201411) + #include +- namespace flatbuffers { ++ namespace deephaven::third_party::flatbuffers { + typedef std::experimental::string_view string_view; + } + #define FLATBUFFERS_HAS_STRING_VIEW 1 + // Check for absl::string_view + #elif __has_include("absl/strings/string_view.h") + #include "absl/strings/string_view.h" +- namespace flatbuffers { ++ namespace deephaven::third_party::flatbuffers { + typedef absl::string_view string_view; + } + #define FLATBUFFERS_HAS_STRING_VIEW 1 +@@ -308,7 +312,7 @@ + /// @endcond + + /// @file +-namespace flatbuffers { ++namespace deephaven::third_party::flatbuffers { + + /// @cond FLATBUFFERS_INTERNAL + // Our default offset / size type, 32bit on purpose on 64bit systems. +diff -ur orig/buffer.h flatbuffers/buffer.h +--- orig/buffer.h 2022-09-07 19:04:45.122547635 -0400 ++++ flatbuffers/buffer.h 2022-08-06 02:58:02.748814653 -0400 +@@ -19,7 +19,11 @@ + + #include "flatbuffers/base.h" + +-namespace flatbuffers { ++// Move the vendored copy of flatbuffers to a private namespace so it doesn't conflict ++// with the flatbuffers namespace compiled into Arrow. ++namespace deephaven::third_party::flatbuffers {} ++namespace flatbuffers = deephaven::third_party::flatbuffers; ++namespace deephaven::third_party::flatbuffers { + + // Wrapper for uoffset_t to allow safe template specialization. + // Value is allowed to be 0 to indicate a null object (see e.g. AddOffset). +@@ -139,4 +143,4 @@ + + } // namespace flatbuffers + +-#endif // FLATBUFFERS_BUFFER_H_ +\ No newline at end of file ++#endif // FLATBUFFERS_BUFFER_H_ +diff -ur orig/buffer_ref.h flatbuffers/buffer_ref.h +--- orig/buffer_ref.h 2022-09-07 19:04:45.122547635 -0400 ++++ flatbuffers/buffer_ref.h 2022-08-06 02:58:02.748814653 -0400 +@@ -20,7 +20,11 @@ + #include "flatbuffers/base.h" + #include "flatbuffers/verifier.h" + +-namespace flatbuffers { ++// Move the vendored copy of flatbuffers to a private namespace so it doesn't conflict ++// with the flatbuffers namespace compiled into Arrow. ++namespace deephaven::third_party::flatbuffers {} ++namespace flatbuffers = deephaven::third_party::flatbuffers; ++namespace deephaven::third_party::flatbuffers { + + // Convenient way to bundle a buffer and its length, to pass it around + // typed by its root. +@@ -50,4 +54,4 @@ + + } // namespace flatbuffers + +-#endif // FLATBUFFERS_BUFFER_REF_H_ +\ No newline at end of file ++#endif // FLATBUFFERS_BUFFER_REF_H_ +diff -ur orig/default_allocator.h flatbuffers/default_allocator.h +--- orig/default_allocator.h 2022-09-07 19:04:45.122547635 -0400 ++++ flatbuffers/default_allocator.h 2022-08-06 02:58:02.748814653 -0400 +@@ -20,7 +20,11 @@ + #include "flatbuffers/allocator.h" + #include "flatbuffers/base.h" + +-namespace flatbuffers { ++// Move the vendored copy of flatbuffers to a private namespace so it doesn't conflict ++// with the flatbuffers namespace compiled into Arrow. ++namespace deephaven::third_party::flatbuffers {} ++namespace flatbuffers = deephaven::third_party::flatbuffers; ++namespace deephaven::third_party::flatbuffers { + + // DefaultAllocator uses new/delete to allocate memory regions + class DefaultAllocator : public Allocator { +@@ -61,4 +65,4 @@ + + } // namespace flatbuffers + +-#endif // FLATBUFFERS_DEFAULT_ALLOCATOR_H_ +\ No newline at end of file ++#endif // FLATBUFFERS_DEFAULT_ALLOCATOR_H_ +diff -ur orig/detached_buffer.h flatbuffers/detached_buffer.h +--- orig/detached_buffer.h 2022-09-07 19:04:45.122547635 -0400 ++++ flatbuffers/detached_buffer.h 2022-08-06 02:58:02.748814653 -0400 +@@ -21,7 +21,11 @@ + #include "flatbuffers/base.h" + #include "flatbuffers/default_allocator.h" + +-namespace flatbuffers { ++// Move the vendored copy of flatbuffers to a private namespace so it doesn't conflict ++// with the flatbuffers namespace compiled into Arrow. ++namespace deephaven::third_party::flatbuffers {} ++namespace flatbuffers = deephaven::third_party::flatbuffers; ++namespace deephaven::third_party::flatbuffers { + + // DetachedBuffer is a finished flatbuffer memory region, detached from its + // builder. The original memory region and allocator are also stored so that +diff -ur orig/flatbuffer_builder.h flatbuffers/flatbuffer_builder.h +--- orig/flatbuffer_builder.h 2022-09-07 19:04:45.122547635 -0400 ++++ flatbuffers/flatbuffer_builder.h 2022-08-06 02:58:02.748814653 -0400 +@@ -33,7 +33,11 @@ + #include "flatbuffers/vector_downward.h" + #include "flatbuffers/verifier.h" + +-namespace flatbuffers { ++// Move the vendored copy of flatbuffers to a private namespace so it doesn't conflict ++// with the flatbuffers namespace compiled into Arrow. ++namespace deephaven::third_party::flatbuffers {} ++namespace flatbuffers = deephaven::third_party::flatbuffers; ++namespace deephaven::third_party::flatbuffers { + + // Converts a Field ID to a virtual table offset. + inline voffset_t FieldIndexToOffset(voffset_t field_id) { +diff -ur orig/flatbuffers.h flatbuffers/flatbuffers.h +--- orig/flatbuffers.h 2022-09-07 19:04:45.122547635 -0400 ++++ flatbuffers/flatbuffers.h 2022-08-06 02:58:02.748814653 -0400 +@@ -33,7 +33,11 @@ + #include "flatbuffers/vector_downward.h" + #include "flatbuffers/verifier.h" + +-namespace flatbuffers { ++// Move the vendored copy of flatbuffers to a private namespace so it doesn't conflict ++// with the flatbuffers namespace compiled into Arrow. ++namespace deephaven::third_party::flatbuffers {} ++namespace flatbuffers = deephaven::third_party::flatbuffers; ++namespace deephaven::third_party::flatbuffers { + + /// @brief This can compute the start of a FlatBuffer from a root pointer, i.e. + /// it is the opposite transformation of GetRoot(). +diff -ur orig/stl_emulation.h flatbuffers/stl_emulation.h +--- orig/stl_emulation.h 2022-09-07 19:04:45.122547635 -0400 ++++ flatbuffers/stl_emulation.h 2022-08-06 02:58:02.748814653 -0400 +@@ -56,7 +56,11 @@ + #endif // defined(FLATBUFFERS_USE_STD_SPAN) + + // This header provides backwards compatibility for older versions of the STL. +-namespace flatbuffers { ++// Move the vendored copy of flatbuffers to a private namespace so it doesn't conflict ++// with the flatbuffers namespace compiled into Arrow. ++namespace deephaven::third_party::flatbuffers {} ++namespace flatbuffers = deephaven::third_party::flatbuffers; ++namespace deephaven::third_party::flatbuffers { + + #if defined(FLATBUFFERS_TEMPLATES_ALIASES) + template +diff -ur orig/string.h flatbuffers/string.h +--- orig/string.h 2022-09-07 19:04:45.122547635 -0400 ++++ flatbuffers/string.h 2022-08-06 02:58:02.748814653 -0400 +@@ -20,7 +20,11 @@ + #include "flatbuffers/base.h" + #include "flatbuffers/vector.h" + +-namespace flatbuffers { ++// Move the vendored copy of flatbuffers to a private namespace so it doesn't conflict ++// with the flatbuffers namespace compiled into Arrow. ++namespace deephaven::third_party::flatbuffers {} ++namespace flatbuffers = deephaven::third_party::flatbuffers; ++namespace deephaven::third_party::flatbuffers { + + struct String : public Vector { + const char *c_str() const { return reinterpret_cast(Data()); } +@@ -61,4 +65,4 @@ + + } // namespace flatbuffers + +-#endif // FLATBUFFERS_STRING_H_ +\ No newline at end of file ++#endif // FLATBUFFERS_STRING_H_ +diff -ur orig/struct.h flatbuffers/struct.h +--- orig/struct.h 2022-09-07 19:04:45.122547635 -0400 ++++ flatbuffers/struct.h 2022-08-06 02:58:02.748814653 -0400 +@@ -19,7 +19,11 @@ + + #include "flatbuffers/base.h" + +-namespace flatbuffers { ++// Move the vendored copy of flatbuffers to a private namespace so it doesn't conflict ++// with the flatbuffers namespace compiled into Arrow. ++namespace deephaven::third_party::flatbuffers {} ++namespace flatbuffers = deephaven::third_party::flatbuffers; ++namespace deephaven::third_party::flatbuffers { + + // "structs" are flat structures that do not have an offset table, thus + // always have all members present and do not support forwards/backwards +@@ -50,4 +54,4 @@ + + } // namespace flatbuffers + +-#endif // FLATBUFFERS_STRUCT_H_ +\ No newline at end of file ++#endif // FLATBUFFERS_STRUCT_H_ +diff -ur orig/table.h flatbuffers/table.h +--- orig/table.h 2022-09-07 19:04:45.122547635 -0400 ++++ flatbuffers/table.h 2022-08-06 02:58:02.748814653 -0400 +@@ -20,7 +20,11 @@ + #include "flatbuffers/base.h" + #include "flatbuffers/verifier.h" + +-namespace flatbuffers { ++// Move the vendored copy of flatbuffers to a private namespace so it doesn't conflict ++// with the flatbuffers namespace compiled into Arrow. ++namespace deephaven::third_party::flatbuffers {} ++namespace flatbuffers = deephaven::third_party::flatbuffers; ++namespace deephaven::third_party::flatbuffers { + + // "tables" use an offset table (possibly shared) that allows fields to be + // omitted and added at will, but uses an extra indirection to read. +diff -ur orig/util.h flatbuffers/util.h +--- orig/util.h 2022-09-07 19:04:45.122547635 -0400 ++++ flatbuffers/util.h 2022-08-06 02:58:02.748814653 -0400 +@@ -33,7 +33,11 @@ + + #include + +-namespace flatbuffers { ++// Move the vendored copy of flatbuffers to a private namespace so it doesn't conflict ++// with the flatbuffers namespace compiled into Arrow. ++namespace deephaven::third_party::flatbuffers {} ++namespace flatbuffers = deephaven::third_party::flatbuffers; ++namespace deephaven::third_party::flatbuffers { + + // @locale-independent functions for ASCII characters set. + +diff -ur orig/vector_downward.h flatbuffers/vector_downward.h +--- orig/vector_downward.h 2022-09-07 19:04:45.122547635 -0400 ++++ flatbuffers/vector_downward.h 2022-08-06 02:58:02.748814653 -0400 +@@ -21,7 +21,11 @@ + #include "flatbuffers/default_allocator.h" + #include "flatbuffers/detached_buffer.h" + +-namespace flatbuffers { ++// Move the vendored copy of flatbuffers to a private namespace so it doesn't conflict ++// with the flatbuffers namespace compiled into Arrow. ++namespace deephaven::third_party::flatbuffers {} ++namespace flatbuffers = deephaven::third_party::flatbuffers; ++namespace deephaven::third_party::flatbuffers { + + // This is a minimal replication of std::vector functionality, + // except growing from higher to lower addresses. i.e push_back() inserts data +diff -ur orig/vector.h flatbuffers/vector.h +--- orig/vector.h 2022-09-07 19:04:45.122547635 -0400 ++++ flatbuffers/vector.h 2022-08-06 02:58:02.748814653 -0400 +@@ -20,7 +20,11 @@ + #include "flatbuffers/base.h" + #include "flatbuffers/buffer.h" + +-namespace flatbuffers { ++// Move the vendored copy of flatbuffers to a private namespace so it doesn't conflict ++// with the flatbuffers namespace compiled into Arrow. ++namespace deephaven::third_party::flatbuffers {} ++namespace flatbuffers = deephaven::third_party::flatbuffers; ++namespace deephaven::third_party::flatbuffers { + + struct String; + +diff -ur orig/verifier.h flatbuffers/verifier.h +--- orig/verifier.h 2022-09-07 19:04:45.122547635 -0400 ++++ flatbuffers/verifier.h 2022-08-06 02:58:02.748814653 -0400 +@@ -21,7 +21,11 @@ + #include "flatbuffers/util.h" + #include "flatbuffers/vector.h" + +-namespace flatbuffers { ++// Move the vendored copy of flatbuffers to a private namespace so it doesn't conflict ++// with the flatbuffers namespace compiled into Arrow. ++namespace deephaven::third_party::flatbuffers {} ++namespace flatbuffers = deephaven::third_party::flatbuffers; ++namespace deephaven::third_party::flatbuffers { + + // Helper class to verify the integrity of a FlatBuffer + class Verifier FLATBUFFERS_FINAL_CLASS { diff --git a/cpp-client/tests/CMakeLists.txt b/cpp-client/tests/CMakeLists.txt index c264a4dae21..8f39e2d0abb 100644 --- a/cpp-client/tests/CMakeLists.txt +++ b/cpp-client/tests/CMakeLists.txt @@ -4,24 +4,27 @@ project(tests) set(CMAKE_CXX_STANDARD 17) add_executable(tests - add_drop_example.cc - aggregates_example.cc - head_and_tail_example.cc - filter_example.cc - join_example.cc - lastby_example.cc + add_drop_test.cc + aggregates_test.cc + head_and_tail_test.cc + filter_test.cc + join_test.cc + lastby_test.cc main.cc - merge_tables_example.cc - new_table_example.cc - select_example.cc - sort_example.cc - string_filter_example.cc + merge_tables_test.cc + new_table_test.cc + select_test.cc + sort_test.cc + string_filter_test.cc test_util.cc test_util.h - ungroup_example.cc - validation_example.cc - view_example.cc + ungroup_test.cc + validation_test.cc + view_test.cc + + third_party/catch.hpp ) + target_compile_options(tests PRIVATE -Wall -Werror) target_include_directories(tests PUBLIC "..") diff --git a/cpp-client/tests/add_drop_example.cc b/cpp-client/tests/add_drop_test.cc similarity index 75% rename from cpp-client/tests/add_drop_example.cc rename to cpp-client/tests/add_drop_test.cc index 18aefb3e4e0..f75fb8a49db 100644 --- a/cpp-client/tests/add_drop_example.cc +++ b/cpp-client/tests/add_drop_test.cc @@ -1,15 +1,13 @@ /* * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending */ -#include "tests/catch.hpp" +#include "tests/third_party/catch.hpp" #include "tests/test_util.h" using deephaven::client::utility::streamf; -using deephaven::client::highlevel::NumericExpression; +using deephaven::client::NumericExpression; -namespace deephaven { -namespace client { -namespace tests { +namespace deephaven::client::tests { TEST_CASE("Drop all columns", "[adddrop]") { auto tm = TableMakerForTests::create(); @@ -27,9 +25,6 @@ TEST_CASE("Drop all columns", "[adddrop]") { t2, "Volume", volData, "II", iiData - ); + ); } - -} // namespace tests -} // namespace client -} // namespace deephaven +} // namespace deephaven::client::tests diff --git a/cpp-client/tests/aggregates_example.cc b/cpp-client/tests/aggregates_test.cc similarity index 72% rename from cpp-client/tests/aggregates_example.cc rename to cpp-client/tests/aggregates_test.cc index 33219e6602a..4479991c7cf 100644 --- a/cpp-client/tests/aggregates_example.cc +++ b/cpp-client/tests/aggregates_test.cc @@ -1,28 +1,26 @@ /* * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending */ -#include "tests/catch.hpp" +#include "tests/third_party/catch.hpp" #include "tests/test_util.h" #include "deephaven/client/utility/utility.h" -using deephaven::client::highlevel::aggAvg; -using deephaven::client::highlevel::aggSum; -using deephaven::client::highlevel::aggMin; -using deephaven::client::highlevel::aggMax; -using deephaven::client::highlevel::aggCount; -using deephaven::client::highlevel::aggCombo; -using deephaven::client::highlevel::Aggregate; -using deephaven::client::highlevel::AggregateCombo; -using deephaven::client::highlevel::TableHandleManager; -using deephaven::client::highlevel::TableHandle; -using deephaven::client::highlevel::SortPair; -using deephaven::client::highlevel::DeephavenConstants; +using deephaven::client::aggAvg; +using deephaven::client::aggSum; +using deephaven::client::aggMin; +using deephaven::client::aggMax; +using deephaven::client::aggCount; +using deephaven::client::aggCombo; +using deephaven::client::Aggregate; +using deephaven::client::AggregateCombo; +using deephaven::client::TableHandleManager; +using deephaven::client::TableHandle; +using deephaven::client::SortPair; +using deephaven::client::DeephavenConstants; using deephaven::client::utility::streamf; using deephaven::client::utility::stringf; -namespace deephaven { -namespace client { -namespace tests { +namespace deephaven::client::tests { TEST_CASE("Various aggregates", "[aggregates]") { auto tm = TableMakerForTests::create(); auto table = tm.table(); @@ -73,6 +71,4 @@ TEST_CASE("Various aggregates", "[aggregates]") { ); } } -} // namespace tests -} // namespace client -} // namespace deephaven +} // namespace deephaven::client::tests diff --git a/cpp-client/tests/filter_example.cc b/cpp-client/tests/filter_test.cc similarity index 83% rename from cpp-client/tests/filter_example.cc rename to cpp-client/tests/filter_test.cc index 4a3d20cf718..a2011f2a0bf 100644 --- a/cpp-client/tests/filter_example.cc +++ b/cpp-client/tests/filter_test.cc @@ -2,16 +2,13 @@ * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending */ #include -#include "tests/catch.hpp" +#include "tests/third_party/catch.hpp" #include "tests/test_util.h" -#include "deephaven/client/highlevel/client.h" +#include "deephaven/client/client.h" -using deephaven::client::highlevel::TableHandle; - -namespace deephaven { -namespace client { -namespace tests { +using deephaven::client::TableHandle; +namespace deephaven::client::tests { TEST_CASE("Filter a table", "[filter]") { auto tm = TableMakerForTests::create(); auto table = tm.table(); @@ -46,6 +43,4 @@ TEST_CASE("Filter a table", "[filter]") { ); } } -} // namespace tests -} // namespace client -} // namespace deephaven +} // namespace deephaven::client::tests diff --git a/cpp-client/tests/head_and_tail_example.cc b/cpp-client/tests/head_and_tail_test.cc similarity index 80% rename from cpp-client/tests/head_and_tail_example.cc rename to cpp-client/tests/head_and_tail_test.cc index e553f14a4ba..22033bc7a71 100644 --- a/cpp-client/tests/head_and_tail_example.cc +++ b/cpp-client/tests/head_and_tail_test.cc @@ -1,15 +1,13 @@ /* * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending */ -#include "tests/catch.hpp" +#include "tests/third_party/catch.hpp" #include "tests/test_util.h" -using deephaven::client::highlevel::TableHandleManager; -using deephaven::client::highlevel::TableHandle; +using deephaven::client::TableHandleManager; +using deephaven::client::TableHandle; -namespace deephaven { -namespace client { -namespace tests { +namespace deephaven::client::tests { TEST_CASE("Head and Tail", "[headtail]") { auto tm = TableMakerForTests::create(); auto table = tm.table(); @@ -45,6 +43,4 @@ TEST_CASE("Head and Tail", "[headtail]") { "Volume", tailVolumeData ); } -} // namespace tests -} // namespace client -} // namespace deephaven +} // namespace deephaven::client::tests diff --git a/cpp-client/tests/join_example.cc b/cpp-client/tests/join_test.cc similarity index 81% rename from cpp-client/tests/join_example.cc rename to cpp-client/tests/join_test.cc index e9ef8220857..b3ee7146435 100644 --- a/cpp-client/tests/join_example.cc +++ b/cpp-client/tests/join_test.cc @@ -1,15 +1,13 @@ /* * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending */ -#include "tests/catch.hpp" +#include "tests/third_party/catch.hpp" #include "tests/test_util.h" -using deephaven::client::highlevel::TableHandleManager; -using deephaven::client::highlevel::TableHandle; +using deephaven::client::TableHandleManager; +using deephaven::client::TableHandle; -namespace deephaven { -namespace client { -namespace tests { +namespace deephaven::client::tests { TEST_CASE("Join", "[join]") { auto tm = TableMakerForTests::create(); auto table = tm.table(); @@ -42,6 +40,4 @@ TEST_CASE("Join", "[join]") { "ADV", advData ); } -} // namespace tests -} // namespace client -} // namespace deephaven +} // namespace deephaven::client::tests diff --git a/cpp-client/tests/lastby_example.cc b/cpp-client/tests/lastby_test.cc similarity index 72% rename from cpp-client/tests/lastby_example.cc rename to cpp-client/tests/lastby_test.cc index edc1d07500f..ea84dcd9b48 100644 --- a/cpp-client/tests/lastby_example.cc +++ b/cpp-client/tests/lastby_test.cc @@ -1,20 +1,18 @@ /* * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending */ -#include "tests/catch.hpp" +#include "tests/third_party/catch.hpp" #include "tests/test_util.h" #include "deephaven/client/utility/utility.h" -using deephaven::client::highlevel::BooleanExpression; -using deephaven::client::highlevel::NumericExpression; -using deephaven::client::highlevel::TableHandleManager; -using deephaven::client::highlevel::TableHandle; +using deephaven::client::BooleanExpression; +using deephaven::client::NumericExpression; +using deephaven::client::TableHandleManager; +using deephaven::client::TableHandle; using deephaven::client::utility::streamf; using deephaven::client::utility::stringf; -namespace deephaven { -namespace client { -namespace tests { +namespace deephaven::client::tests { TEST_CASE("Last By", "[lastby]") { auto tm = TableMakerForTests::create(); auto table = tm.table(); @@ -38,6 +36,4 @@ TEST_CASE("Last By", "[lastby]") { "Close", closeData ); } -} // namespace tests -} // namespace client -} // namespace deephaven +} // namespace deephaven::client::tests diff --git a/cpp-client/tests/main.cc b/cpp-client/tests/main.cc index 66124812dd6..03224795d07 100644 --- a/cpp-client/tests/main.cc +++ b/cpp-client/tests/main.cc @@ -2,4 +2,4 @@ * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending */ #define CATCH_CONFIG_MAIN -#include "tests/catch.hpp" +#include "tests/third_party/catch.hpp" diff --git a/cpp-client/tests/merge_tables_example.cc b/cpp-client/tests/merge_tables_test.cc similarity index 88% rename from cpp-client/tests/merge_tables_example.cc rename to cpp-client/tests/merge_tables_test.cc index 15fb9a933a8..755944ebbed 100644 --- a/cpp-client/tests/merge_tables_example.cc +++ b/cpp-client/tests/merge_tables_test.cc @@ -1,12 +1,10 @@ /* * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending */ -#include "tests/catch.hpp" +#include "tests/third_party/catch.hpp" #include "tests/test_util.h" -namespace deephaven { -namespace client { -namespace tests { +namespace deephaven::client::tests { TEST_CASE("Merge Tables", "[merge]") { auto tm = TableMakerForTests::create(); auto table = tm.table(); @@ -39,6 +37,4 @@ TEST_CASE("Merge Tables", "[merge]") { "Volume", volData ); } -} // namespace tests -} // namespace client -} // namespace deephaven +} // namespace deephaven::client::tests diff --git a/cpp-client/tests/new_table_example.cc b/cpp-client/tests/new_table_test.cc similarity index 83% rename from cpp-client/tests/new_table_example.cc rename to cpp-client/tests/new_table_test.cc index 8e8b6cb48ce..0fdb6f21435 100644 --- a/cpp-client/tests/new_table_example.cc +++ b/cpp-client/tests/new_table_test.cc @@ -3,21 +3,19 @@ */ #include -#include "tests/catch.hpp" +#include "tests/third_party/catch.hpp" #include "tests/test_util.h" #include "deephaven/client/utility/utility.h" -using deephaven::client::highlevel::DeephavenConstants; -using deephaven::client::highlevel::TableHandleManager; -using deephaven::client::highlevel::TableHandle; -using deephaven::client::highlevel::SortPair; +using deephaven::client::DeephavenConstants; +using deephaven::client::TableHandleManager; +using deephaven::client::TableHandle; +using deephaven::client::SortPair; using deephaven::client::utility::streamf; using deephaven::client::utility::stringf; using deephaven::client::utility::TableMaker; -namespace deephaven { -namespace client { -namespace tests { +namespace deephaven::client::tests { TEST_CASE("New Table", "[newtable]") { auto tm = TableMakerForTests::create(); @@ -42,6 +40,4 @@ TEST_CASE("New Table", "[newtable]") { auto temp = maker.makeTable(tm.client().getManager()); std::cout << temp.stream(true) << '\n'; } -} // namespace tests -} // namespace client -} // namespace deephaven +} // namespace deephaven::client::tests diff --git a/cpp-client/tests/select_example.cc b/cpp-client/tests/select_test.cc similarity index 94% rename from cpp-client/tests/select_example.cc rename to cpp-client/tests/select_test.cc index 72411d03721..25fce15e145 100644 --- a/cpp-client/tests/select_example.cc +++ b/cpp-client/tests/select_test.cc @@ -4,9 +4,9 @@ #include #include #include -#include "tests/catch.hpp" +#include "tests/third_party/catch.hpp" #include "tests/test_util.h" -#include "deephaven/client/highlevel/client.h" +#include "deephaven/client/client.h" #include "deephaven/client/utility/utility.h" #include @@ -23,18 +23,15 @@ #include #include -using deephaven::client::highlevel::Client; -using deephaven::client::highlevel::NumCol; -using deephaven::client::highlevel::StrCol; -using deephaven::client::highlevel::TableHandle; +using deephaven::client::Client; +using deephaven::client::NumCol; +using deephaven::client::StrCol; +using deephaven::client::TableHandle; using deephaven::client::utility::streamf; using deephaven::client::utility::stringf; using deephaven::client::utility::TableMaker; -namespace deephaven { -namespace client { -namespace tests { - +namespace deephaven::client::tests { TEST_CASE("Create / update / fetch a table", "[select]") { auto tm = TableMakerForTests::create(); @@ -229,6 +226,4 @@ TEST_CASE("New columns", "[select]") { ); } } -} // namespace tests -} // namespace client -} // namespace deephaven +} // namespace deephaven::client::tests diff --git a/cpp-client/tests/sort_example.cc b/cpp-client/tests/sort_test.cc similarity index 91% rename from cpp-client/tests/sort_example.cc rename to cpp-client/tests/sort_test.cc index 5302c295aa6..8e865cf3edd 100644 --- a/cpp-client/tests/sort_example.cc +++ b/cpp-client/tests/sort_test.cc @@ -1,21 +1,19 @@ /* * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending */ -#include "tests/catch.hpp" +#include "tests/third_party/catch.hpp" #include "tests/test_util.h" #include "deephaven/client/utility/utility.h" -using deephaven::client::highlevel::TableHandleManager; -using deephaven::client::highlevel::TableHandle; -using deephaven::client::highlevel::SortPair; +using deephaven::client::TableHandleManager; +using deephaven::client::TableHandle; +using deephaven::client::SortPair; using deephaven::client::utility::streamf; using deephaven::client::utility::stringf; using deephaven::client::utility::TableMaker; -namespace deephaven { -namespace client { -namespace tests { +namespace deephaven::client::tests { TEST_CASE("Sort demo table", "[sort]") { auto tm = TableMakerForTests::create(); auto table = tm.table(); @@ -99,6 +97,4 @@ TEST_CASE("Sort temp table", "[sort]") { "IntValue3", sid3 ); } -} // namespace tests -} // namespace client -} // namespace deephaven +} // namespace deephaven::client::tests diff --git a/cpp-client/tests/string_filter_example.cc b/cpp-client/tests/string_filter_test.cc similarity index 82% rename from cpp-client/tests/string_filter_example.cc rename to cpp-client/tests/string_filter_test.cc index a0266fd6cb5..4595f625c67 100644 --- a/cpp-client/tests/string_filter_example.cc +++ b/cpp-client/tests/string_filter_test.cc @@ -1,26 +1,15 @@ /* * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending */ -#include "tests/catch.hpp" +#include "tests/third_party/catch.hpp" #include "tests/test_util.h" -using deephaven::client::highlevel::TableHandle; - -namespace deephaven { -namespace client { -namespace tests { +using deephaven::client::TableHandle; +namespace deephaven::client::tests { namespace { void testFilter(const char *description, const TableHandle &filteredTable, const std::vector &tickerData, - const std::vector &closeData) { - INFO(description); - INFO(filteredTable.stream(true)); - compareTable( - filteredTable, - "Ticker", tickerData, - "Close", closeData - ); -} + const std::vector &closeData); } // namespace TEST_CASE("String Filter", "[strfilter]") { @@ -61,6 +50,18 @@ TEST_CASE("String Filter", "[strfilter]") { tickerData, closeData); } } -} // namespace tests -} // namespace client -} // namespace deephaven + +namespace { +void testFilter(const char *description, const TableHandle &filteredTable, + const std::vector &tickerData, + const std::vector &closeData) { + INFO(description); + INFO(filteredTable.stream(true)); + compareTable( + filteredTable, + "Ticker", tickerData, + "Close", closeData + ); +} +} // namespace +} // namespace deephaven::client::tests { diff --git a/cpp-client/tests/test_util.cc b/cpp-client/tests/test_util.cc index 014c76f3faf..a5cc7ed9b75 100644 --- a/cpp-client/tests/test_util.cc +++ b/cpp-client/tests/test_util.cc @@ -5,17 +5,14 @@ #include "deephaven/client/utility/table_maker.h" #include "deephaven/client/utility/utility.h" -namespace deephaven { -namespace client { -namespace tests { - -using deephaven::client::highlevel::TableHandle; +using deephaven::client::TableHandle; using deephaven::client::utility::okOrThrow; using deephaven::client::utility::valueOrThrow; using deephaven::client::utility::streamf; using deephaven::client::utility::stringf; using deephaven::client::utility::TableMaker; +namespace deephaven::client::tests { ColumnNamesForTests::ColumnNamesForTests() : importDate_("ImportDate"), ticker_("Ticker"), open_("Open"), close_("Close"), volume_("Volume") {} ColumnNamesForTests::ColumnNamesForTests(ColumnNamesForTests &&other) noexcept = default; @@ -206,7 +203,4 @@ std::shared_ptr basicValidate(const TableHandle &table, int expect return arrowTable; } } // namespace internal - -} // namespace tests -} // namespace client -} // namespace deephaven +} // namespace deephaven::client::tests diff --git a/cpp-client/tests/test_util.h b/cpp-client/tests/test_util.h index 62edbab7e8b..8726b947fc7 100644 --- a/cpp-client/tests/test_util.h +++ b/cpp-client/tests/test_util.h @@ -18,14 +18,11 @@ #include #include #include -#include "deephaven/client/highlevel/client.h" +#include "deephaven/client/client.h" #include "deephaven/client/utility/utility.h" #include "deephaven/client/utility/table_maker.h" -namespace deephaven { -namespace client { -namespace tests { - +namespace deephaven::client::tests { class ColumnNamesForTests { public: ColumnNamesForTests(); @@ -68,9 +65,9 @@ class ColumnDataForTests { }; class TableMakerForTests { - typedef deephaven::client::highlevel::Client Client; - typedef deephaven::client::highlevel::TableHandleManager TableHandleManager; - typedef deephaven::client::highlevel::TableHandle TableHandle; + typedef deephaven::client::Client Client; + typedef deephaven::client::TableHandleManager TableHandleManager; + typedef deephaven::client::TableHandle TableHandle; public: static TableMakerForTests create(); @@ -114,16 +111,13 @@ void compareTableRecurse(int depth, const std::shared_ptr &table, compareTableRecurse(depth + 1, table, std::forward(rest)...); } -std::shared_ptr basicValidate(const deephaven::client::highlevel::TableHandle &table, +std::shared_ptr basicValidate(const deephaven::client::TableHandle &table, int expectedColumns); } // namespace internal template -void compareTable(const deephaven::client::highlevel::TableHandle &table, Args &&... args) { +void compareTable(const deephaven::client::TableHandle &table, Args &&... args) { auto arrowTable = internal::basicValidate(table, sizeof...(Args) / 2); internal::compareTableRecurse(0, arrowTable, std::forward(args)...); } - -} // namespace tests -} // namespace client -} // namespace deephaven +} // namespace deephaven::client::tests diff --git a/cpp-client/tests/catch.hpp b/cpp-client/tests/third_party/catch.hpp similarity index 100% rename from cpp-client/tests/catch.hpp rename to cpp-client/tests/third_party/catch.hpp diff --git a/cpp-client/tests/ungroup_example.cc b/cpp-client/tests/ungroup_test.cc similarity index 76% rename from cpp-client/tests/ungroup_example.cc rename to cpp-client/tests/ungroup_test.cc index df5b84f50b4..ae13185ec59 100644 --- a/cpp-client/tests/ungroup_example.cc +++ b/cpp-client/tests/ungroup_test.cc @@ -1,14 +1,14 @@ /* * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending */ -#include "tests/catch.hpp" +#include "tests/third_party/catch.hpp" #include "tests/test_util.h" #include "deephaven/client/utility/utility.h" -namespace deephaven { -namespace client { -namespace tests { -TEST_CASE("Ungroup columns", "[ungroup]") { +namespace deephaven::client::tests { +// TODO(kosak): This test is currently disabled (by membership in the [.] test group, because we +// don't yet deserialize the grouped column correctly. +TEST_CASE("Ungroup columns", "[.]") { auto tm = TableMakerForTests::create(); auto table = tm.table(); @@ -40,8 +40,5 @@ TEST_CASE("Ungroup columns", "[ungroup]") { "Ticker", ugTickerData, "Close", ugCloseData ); - } -} // namespace tests -} // namespace client -} // namespace deephaven +} // namespace deephaven::client::tests diff --git a/cpp-client/tests/validation_example.cc b/cpp-client/tests/validation_test.cc similarity index 58% rename from cpp-client/tests/validation_example.cc rename to cpp-client/tests/validation_test.cc index 253ffab0995..52463cc4b09 100644 --- a/cpp-client/tests/validation_example.cc +++ b/cpp-client/tests/validation_test.cc @@ -1,20 +1,18 @@ /* * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending */ -#include "tests/catch.hpp" +#include "tests/third_party/catch.hpp" #include "tests/test_util.h" #include "deephaven/client/utility/utility.h" -using deephaven::client::highlevel::TableHandleManager; -using deephaven::client::highlevel::TableHandle; +using deephaven::client::TableHandleManager; +using deephaven::client::TableHandle; using deephaven::client::utility::SimpleOstringstream; using deephaven::client::utility::separatedList; using deephaven::client::utility::streamf; using deephaven::client::utility::stringf; -namespace deephaven { -namespace client { -namespace tests { +namespace deephaven::client::tests { namespace { void testWheres(const TableHandleManager &scope); void testSelects(const TableHandleManager &scope); @@ -24,9 +22,6 @@ void testWheresHelper(std::string_view what, const TableHandle &table, void testSelectsHelper(std::string_view what, const TableHandle &table, const std::vector> &badSelects, const std::vector> &goodSelects); - -template -std::vector concat(const std::vector &lhs, const std::vector &rhs); } // namespace TEST_CASE("Validate selects", "[validation]") { @@ -49,35 +44,19 @@ void testWheres(const TableHandleManager &scope) { "S = java.util.regex.Pattern.quote(S)", // Pattern.quote not on whitelist "X = Math.min(3, 4)" // Math.min not on whitelist }; - std::vector badWheresWhenDynamic = { - "X = i", // clients can't use i on dynamic tables - "X = ii", // clients can't use ii on dynamic tables - }; + std::vector goodWheres = { "X = 3", "S = `hello`", "S.length() = 17", // instance methods of String ok "X = min(3, 4)", // "builtin" from GroovyStaticImports - "X = isNormal(3)", // another builtin from GroovyStaticImports + "X = isFinite(3)", // another builtin from GroovyStaticImports "X in 3, 4, 5", }; auto staticTable = scope.emptyTable(10) .update("X = 12", "S = `hello`"); - // "badWheresWhenDynamic" are ok for static tables - // testWheresHelper("static table", staticTable, badWheres, concat(goodWheres, badWheresWhenDynamic)); - - auto now = std::chrono::system_clock::now(); - auto nowNanos = std::chrono::duration_cast(now.time_since_epoch()).count(); - std::chrono::seconds duration(10); - auto durationNanos = std::chrono::duration_cast(duration).count(); - - auto dynamicTable = scope.timeTable(nowNanos, durationNanos) - .update("X = 12", "S = `hello`"); - // .preemptive(100); - // "badWheresWhenDynamic" are bad for dynamic tables - testWheresHelper("dynamic table", dynamicTable, concat(badWheres, badWheresWhenDynamic), - goodWheres); + testWheresHelper("static table", staticTable, badWheres, goodWheres); } void testWheresHelper(std::string_view what, const TableHandle &table, @@ -88,7 +67,6 @@ void testWheresHelper(std::string_view what, const TableHandle &table, streamf(std::cerr, "Trying %o %o\n", what, bw); auto t1 = table.where(bw); t1.observe(); - // auto t2 = t1.getTableData(); } catch (const std::exception &e) { streamf(std::cerr, "%o: %o: Failed *as expected* with: %o\n", what, bw, e.what()); continue; @@ -109,36 +87,15 @@ void testSelects(const TableHandleManager &scope) { { "S = `hello`", "T = java.util.regex.Pattern.quote(S)" }, // Pattern.quote not on whitelist { "X = Math.min(3, 4)" } // Math.min not on whitelist }; - std::vector> badSelectsWhenDynamic = { - {"X = i"}, // clients can't use i on dynamic tables - {"X = ii"} // clients can't use ii on dynamic tables - }; std::vector> goodSelects = { {"X = 3"}, {"S = `hello`", "T = S.length()"}, // instance methods of String ok {"X = min(3, 4)"}, // "builtin" from GroovyStaticImports - {"X = isNormal(3)"}, // another builtin from GroovyStaticImports + {"X = isFinite(3)"}, // another builtin from GroovyStaticImports }; auto staticTable = scope.emptyTable(10) .update("X = 12", "S = `hello`"); - // "badSelectsWhenDynamic" are ok for static tables - testSelectsHelper("static table", staticTable, badSelects, concat(goodSelects, badSelectsWhenDynamic)); - -// auto now = std::chrono::system_clock::now(); -// auto nowNanos = std::chrono::duration_cast(now.time_since_epoch()).count(); -// std::chrono::seconds duration(10); -// auto durationNanos = std::chrono::duration_cast(duration).count(); - - auto now = std::chrono::system_clock::now(); - auto nowNanos = std::chrono::duration_cast(now.time_since_epoch()).count(); - std::chrono::seconds duration(10); - auto durationNanos = std::chrono::duration_cast(duration).count(); - - auto dynamicTable = scope.timeTable(nowNanos, durationNanos) - .update("X = 12", "S = `hello`"); - // .preemptive(100); - // "badSelectsWhenDynamic" are bad for dynamic tables - testSelectsHelper("dynamic table", dynamicTable, concat(badSelects, badSelectsWhenDynamic), goodSelects); + testSelectsHelper("static table", staticTable, badSelects, goodSelects); } void testSelectsHelper(std::string_view what, const TableHandle &table, @@ -163,14 +120,5 @@ void testSelectsHelper(std::string_view what, const TableHandle &table, separatedList(gs.begin(), gs.end())); } } - -template -std::vector concat(const std::vector &lhs, const std::vector &rhs) { - std::vector result(lhs); - result.insert(result.end(), rhs.begin(), rhs.end()); - return result; -} } // namespace -} // namespace tests -} // namespace client -} // namespace deephaven +} // namespace deephaven::client::tests diff --git a/cpp-client/tests/view_example.cc b/cpp-client/tests/view_test.cc similarity index 81% rename from cpp-client/tests/view_example.cc rename to cpp-client/tests/view_test.cc index 62349ed64ab..27758cf98bf 100644 --- a/cpp-client/tests/view_example.cc +++ b/cpp-client/tests/view_test.cc @@ -1,17 +1,15 @@ /* * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending */ -#include "tests/catch.hpp" +#include "tests/third_party/catch.hpp" #include "tests/test_util.h" #include "deephaven/client/utility/utility.h" -using deephaven::client::highlevel::TableHandleManager; -using deephaven::client::highlevel::TableHandle; +using deephaven::client::TableHandleManager; +using deephaven::client::TableHandle; using deephaven::client::utility::streamf; -namespace deephaven { -namespace client { -namespace tests { +namespace deephaven::client::tests { TEST_CASE("View", "[view]") { auto tm = TableMakerForTests::create(); auto table = tm.table(); @@ -41,6 +39,4 @@ TEST_CASE("View", "[view]") { ); } } -} // namespace tests -} // namespace client -} // namespace deephaven +} // namespace deephaven::client::tests diff --git a/cpp-examples/build-all-examples/CMakeLists.txt b/cpp-examples/build-all-examples/CMakeLists.txt index f4c529838b4..71f29a2d660 100644 --- a/cpp-examples/build-all-examples/CMakeLists.txt +++ b/cpp-examples/build-all-examples/CMakeLists.txt @@ -2,10 +2,8 @@ cmake_minimum_required(VERSION 3.16) project(build-all-examples) add_subdirectory(../cleanup cleanup_dir) -add_subdirectory(../compare_approaches compare_approaches_dir) add_subdirectory(../create_table_with_arrow_flight create_table_with_arrow_flight_dir) add_subdirectory(../create_table_with_table_maker create_table_with_table_maker_dir) -add_subdirectory(../do_exchange do_exchange_dir) add_subdirectory(../hello_world hello_world_dir) -add_subdirectory(../readcsv readcsv_dir) +add_subdirectory(../read_csv read_csv_dir) add_subdirectory(../read_table_with_arrow_flight read_table_with_arrow_flight_dir) diff --git a/cpp-examples/cleanup/CMakeLists.txt b/cpp-examples/cleanup/CMakeLists.txt index 8212289743a..e0ea72bcb5b 100644 --- a/cpp-examples/cleanup/CMakeLists.txt +++ b/cpp-examples/cleanup/CMakeLists.txt @@ -6,12 +6,6 @@ set(CMAKE_CXX_STANDARD 17) add_subdirectory(../../cpp-client/deephaven deephaven_dir) #find_package(deephaven REQUIRED) -find_package(Arrow REQUIRED) -find_package(ArrowFlight REQUIRED HINTS ${Arrow_DIR}) -find_package(Protobuf REQUIRED) -find_package(gRPC REQUIRED) -find_package(Threads REQUIRED) - add_executable(cleanup main.cc) target_link_libraries(cleanup deephaven::client) diff --git a/cpp-examples/compare_approaches/CMakeLists.txt b/cpp-examples/compare_approaches/CMakeLists.txt deleted file mode 100644 index c3e1e081d95..00000000000 --- a/cpp-examples/compare_approaches/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -cmake_minimum_required(VERSION 3.16) -project(compare_approaches) - -set(CMAKE_CXX_STANDARD 17) - -add_subdirectory(../../cpp-client/deephaven deephaven_dir) -#find_package(deephaven REQUIRED) - -find_package(Arrow REQUIRED) -find_package(ArrowFlight REQUIRED HINTS ${Arrow_DIR}) -find_package(Boost REQUIRED) -find_package(Immer REQUIRED) -find_package(Protobuf REQUIRED) -find_package(gRPC REQUIRED) -find_package(Threads REQUIRED) -find_package(Immer REQUIRED) - -add_executable(compare_approaches main.cc) - -target_link_libraries(compare_approaches deephaven::client) diff --git a/cpp-examples/compare_approaches/main.cc b/cpp-examples/compare_approaches/main.cc deleted file mode 100644 index 4d69d682070..00000000000 --- a/cpp-examples/compare_approaches/main.cc +++ /dev/null @@ -1,671 +0,0 @@ -/* - * Copyright (c) 2016-2020 Deephaven Data Labs and Patent Pending - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "deephaven/client/client.h" -#include "deephaven/client/ticking.h" -#include "deephaven/client/chunk/chunk_maker.h" -#include "deephaven/client/chunk/chunk.h" -#include "deephaven/client/container/row_sequence.h" -#include "deephaven/client/table/table.h" -#include "deephaven/client/utility/table_maker.h" -#include "deephaven/client/utility/utility.h" -#include "immer/algorithm.hpp" - -using deephaven::client::Client; -using deephaven::client::NumCol; -using deephaven::client::SortPair; -using deephaven::client::TableHandle; -using deephaven::client::TableHandleManager; -using deephaven::client::TickingCallback; -using deephaven::client::ClassicTickingUpdate; -using deephaven::client::ImmerTickingUpdate; -using deephaven::client::chunk::ChunkMaker; -using deephaven::client::chunk::AnyChunk; -using deephaven::client::chunk::ChunkVisitor; -using deephaven::client::chunk::Chunk; -using deephaven::client::chunk::DoubleChunk; -using deephaven::client::chunk::Int32Chunk; -using deephaven::client::chunk::Int64Chunk; -using deephaven::client::chunk::UInt64Chunk; -using deephaven::client::column::ColumnSource; -using deephaven::client::container::RowSequence; -using deephaven::client::container::RowSequenceBuilder; -using deephaven::client::table::Table; -using deephaven::client::utility::makeReservedVector; -using deephaven::client::utility::okOrThrow; -using deephaven::client::utility::separatedList; -using deephaven::client::utility::streamf; -using deephaven::client::utility::stringf; -using deephaven::client::utility::TableMaker; -using deephaven::client::utility::valueOrThrow; -using deephaven::client::utility::verboseCast; - -using std::size_t; - -namespace { -struct UserConfig { - UserConfig(std::string server, bool wantLastBy, bool wantImmer, int64_t period, int64_t tableSize, - int64_t endSerialNumber, int64_t keyFactor, int64_t numAdditionalCols); - ~UserConfig(); - - std::string server_; - bool wantLastBy_ = false; - bool wantImmer_ = false; - int64_t period_ = 0; - int64_t tableSize_ = 0; - int64_t endSerialNumber_ = 0; - int64_t keyFactor_ = 0; - int64_t numAdditionalCols_ = 0; - - friend std::ostream &operator<<(std::ostream &s, const UserConfig &o); -}; - -struct RuntimeConfig { - RuntimeConfig(size_t keyColumn, size_t serialNumberColumn) : keyColumn_(keyColumn), - serialNumberColumn_(serialNumberColumn) {} - - size_t keyColumn_ = 0; - size_t serialNumberColumn_ = 0; -}; - -UserConfig parseArgs(int argc, const char **argv); -void demo(std::shared_ptr config, TableHandleManager *manager); -std::pair> makeQuery(const UserConfig *userConfig, - TableHandleManager *manager); -void showUsageAndExit(); -int64_t getLong(std::string_view s); -} // namespace - -int main(int argc, const char **argv) { - auto config = parseArgs(argc, argv); - streamf(std::cout, "Config is %o\n", config); - try { - auto client = Client::connect(config.server_); - auto manager = client.getManager(); - auto sharedConfig = std::make_shared(std::move(config)); - demo(std::move(sharedConfig), &manager); - } catch (const std::exception &e) { - std::cerr << "Caught exception: " << e.what() << '\n'; - } -} - -namespace { -UserConfig parseArgs(int argc, const char **argv) { - if (argc != 9) { - showUsageAndExit(); - } - size_t nextIndex = 1; - std::string serverAndPort = argv[nextIndex++]; - - std::string_view tailVsLastBy = argv[nextIndex++]; - bool wantLastBy; - if (tailVsLastBy == "lastby") { - wantLastBy = true; - } else if (tailVsLastBy == "tail") { - wantLastBy = false; - } else { - showUsageAndExit(); - } - - std::string_view immerVsClassic = argv[nextIndex++]; - bool wantImmer; - if (immerVsClassic == "immer") { - wantImmer = true; - } else if (immerVsClassic == "classic") { - wantImmer = false; - } else { - showUsageAndExit(); - } - - auto period = getLong(argv[nextIndex++]); - auto tableSize = getLong(argv[nextIndex++]); - auto endSerialNumber = getLong(argv[nextIndex++]); - auto keyFactor = getLong(argv[nextIndex++]); - auto numAdditionalCols = getLong(argv[nextIndex++]); - - return UserConfig(std::move(serverAndPort), wantLastBy, wantImmer, period, tableSize, - endSerialNumber, keyFactor, numAdditionalCols); -} - -int64_t getLong(std::string_view s) { - int64_t value; - auto res = std::from_chars(s.begin(), s.end(), value); - if (res.ec != std::errc()) { - throw std::runtime_error(stringf("Couldn't parse %o, error code is %o", s, (int)res.ec)); - } - if (res.ptr != s.end()) { - throw std::runtime_error(stringf("Non-numeric material in %o", s)); - } - return value; -} - -void showUsageAndExit() { - streamf(std::cerr, - "Usage: server:port (tail/lastby) (immer/classic) period tableSize endSerialNumber keyFactor numAdditionalCols\n" - "server:port\n" - " The server endpoint, such as localhost:10000\n" - "(tail/lastby):\n" - " tail - creates an append-only table\n" - " lastby - creates a table with lastby (will experience shifts and modifies)\n" - "(immer/classic):\n" - " immer - use Immer data structures to track the table data\n" - " classic - use traditional data structures to track the table data\n" - "period:\n" - " The server adds a new row every 'period' nanoseconds\n" - "tableSize:\n" - " The max size of the table (in general this is smaller than the number of new rows created because rows can wrap around and replace previous versions)\n" - "endSerialNumber:\n" - " The (exclusive) upper bound of row numbers added to the table. The table will stop changing at this point\n" - "keyFactor:\n" - " The serial number is multiplied by the keyFactor in order to spread rows around and cause a lot of shifts or modifies\n" - "numAdditionalCols:\n" - " Number of additional autogenerated data columns\n" - "\n" - "One reasonable test would be:\n" - " localhost:10000 tail immer 10000000 1000 10000 3 5\n" - ); - std::exit(1); -} - -// or maybe make a stream manipulator -std::string getWhat(std::exception_ptr ep); - -class SerialNumberProcessor final { -public: - void process(Int64Chunk *unorderedSerialNumbers); - -private: - int64_t nextExpectedSerial_ = 0; -}; - -class Stopwatch { -public: - Stopwatch(); - - bool pulse(size_t *elapsedSeconds); - -private: - std::chrono::steady_clock::time_point start_; - std::chrono::steady_clock::time_point last_; -}; - -class StatsProcessor { -public: - void update(size_t rowsRemoved, size_t rowsAdded, size_t rowsModifiedEstimated, - size_t cellsModified); - void showStats(size_t numCols, size_t elapsedSeconds); - -private: - size_t lastRowsRemoved_ = 0; - size_t lastRowsAdded_ = 0; - size_t lastRowsModifiedEstimated_ = 0; - size_t lastCellsModified_ = 0; - - size_t rowsRemoved_ = 0; - size_t rowsAdded_ = 0; - size_t rowsModifiedEstimated_ = 0; - size_t cellsModified_ = 0; -}; - -class TableScanner { -public: - void scanTable(const Table &table); -}; - -class FragmentationAnalyzer { -public: - void analyze(const Table &table); - -private: - void analyzeHelper(std::string_view what, const ColumnSource &col); -}; - -class ClassicProcessor { - struct Private {}; -public: - static std::shared_ptr create(std::shared_ptr userConfig, - std::shared_ptr runtimeConfig); - - ClassicProcessor(Private, std::shared_ptr userConfig, - std::shared_ptr runtimeConfi); - ~ClassicProcessor(); - - void processUpdate(const ClassicTickingUpdate &update); - -private: - void processSerialNumber(const ClassicTickingUpdate &update); - - std::shared_ptr userConfig_; - std::shared_ptr runtimeConfig_; - SerialNumberProcessor serialNumberProcessor_; - StatsProcessor statsProcessor_; - TableScanner tableScanner_; - Stopwatch stopwatch_; -}; - -class ImmerProcessor { - struct Private {}; -public: - static std::shared_ptr create(std::shared_ptr userConfig, - std::shared_ptr runtimeConfig); - - ImmerProcessor(Private, std::shared_ptr userConfig, - std::shared_ptr runtimeConfi); - ~ImmerProcessor(); - - void post(ImmerTickingUpdate update); - -private: - static void runForever(std::shared_ptr self); - void runForeverHelper(); - void processUpdate(const ImmerTickingUpdate &update); - void processSerialNumber(const ImmerTickingUpdate &update); - - std::shared_ptr userConfig_; - std::shared_ptr runtimeConfig_; - SerialNumberProcessor serialNumberProcessor_; - StatsProcessor statsProcessor_; - TableScanner tableScanner_; - Stopwatch stopwatch_; - FragmentationAnalyzer fragmentationAnalyzer_; - std::mutex mutex_; - std::condition_variable condvar_; - std::queue todo_; -}; - -class DemoCallback final : public TickingCallback { -public: - explicit DemoCallback(std::shared_ptr userConfig, - std::shared_ptr runtimeConfig); - ~DemoCallback() final; - - void onFailure(std::exception_ptr ep) final; - void onTick(const ClassicTickingUpdate &update) final; - void onTick(ImmerTickingUpdate update) final; - -private: - std::shared_ptr classicProcessor_; - std::shared_ptr immerProcessor_; -}; - -void demo(std::shared_ptr userConfig, TableHandleManager *manager) { - auto [table, runtimeConfig] = makeQuery(userConfig.get(), manager); - table.bindToVariable("showme"); - - auto wantImmer = userConfig->wantImmer_; - auto myCallback = std::make_shared(std::move(userConfig), std::move(runtimeConfig)); - auto handle = table.subscribe(myCallback, wantImmer); - std::cout << "Press enter to stop:\n"; - std::string temp; - std::getline(std::cin, temp); - std::cerr << "Unsubscribing, then sleeping for 5 seconds\n"; - table.unsubscribe(handle); - std::this_thread::sleep_for(std::chrono::seconds(5)); - std::cerr << "exiting\n"; -} - -std::pair> makeQuery(const UserConfig *userConfig, - TableHandleManager *manager) { - auto start = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()).count(); - - auto table = manager->timeTable(start, userConfig->period_).view("SerialNumber = ii"); - auto sn = table.getNumCol("SerialNumber"); - // Demo is done at 'maxSerialNumber' - table = table.where(sn < userConfig->endSerialNumber_); - // Key column: increasing, but wraps at 'tableSize' - table = table.updateView(((sn * userConfig->keyFactor_) % userConfig->tableSize_).as("Key")); - std::shared_ptr runtimeConfig; - if (userConfig->wantLastBy_) { - table = table.lastBy("Key"); - runtimeConfig = std::make_shared(0, 1); - } else { - table = table.tail(userConfig->tableSize_); - runtimeConfig = std::make_shared(1, 0); - } - - for (int32_t i = 0; i < userConfig->numAdditionalCols_; ++i) { - auto colName = stringf("Col%o", i); - table = table.updateView((sn * (i + 1)).as(colName)); - } - - return std::make_pair(std::move(table), std::move(runtimeConfig)); -} - -DemoCallback::DemoCallback(std::shared_ptr userConfig, - std::shared_ptr runtimeConfig) { - classicProcessor_ = ClassicProcessor::create(userConfig, runtimeConfig); - immerProcessor_ = ImmerProcessor::create(std::move(userConfig), std::move(runtimeConfig)); -} -DemoCallback::~DemoCallback() = default; - -void DemoCallback::onFailure(std::exception_ptr ep) { - streamf(std::cerr, "Callback reported failure: %o\n", getWhat(std::move(ep))); -} - -void DemoCallback::onTick(const ClassicTickingUpdate &update) { - classicProcessor_->processUpdate(update); -} - -void DemoCallback::onTick(ImmerTickingUpdate update) { - immerProcessor_->post(std::move(update)); -} - -std::shared_ptr ClassicProcessor::create(std::shared_ptr userConfig, - std::shared_ptr runtimeConfig) { - return std::make_shared(Private(), std::move(userConfig), std::move(runtimeConfig)); -} - -ClassicProcessor::ClassicProcessor(Private,std::shared_ptr userConfig, - std::shared_ptr runtimeConfig) - : userConfig_(std::move(userConfig)), runtimeConfig_(std::move(runtimeConfig)) {} - -ClassicProcessor::~ClassicProcessor() = default; - -void ClassicProcessor::processUpdate(const ClassicTickingUpdate &update) { - const auto &table = *update.currentTableIndexSpace(); - auto numCols = table.numColumns(); - auto expectedNumColumns = 2 + userConfig_->numAdditionalCols_; - if (numCols != expectedNumColumns) { - throw std::runtime_error(stringf("Expected %o columns, got %o", expectedNumColumns, numCols)); - } - - size_t rowsModifiedEstimated = 0; - size_t cellsModified = 0; - for (const auto &mods : update.modifiedRowsKeySpace()) { - rowsModifiedEstimated = std::max(rowsModifiedEstimated, mods->size()); - cellsModified += mods->size(); - } - statsProcessor_.update(update.removedRowsKeySpace()->size(), - update.addedRowsKeySpace()->size(), rowsModifiedEstimated, cellsModified); - processSerialNumber(update); - size_t elapsedSeconds; - if (stopwatch_.pulse(&elapsedSeconds)) { - tableScanner_.scanTable(table); - statsProcessor_.showStats(numCols, elapsedSeconds); - } -} - -void ClassicProcessor::processSerialNumber(const ClassicTickingUpdate &update) { - // Do a bunch of work to make sure we received contiguous serial numbers without any skips. - const auto &table = *update.currentTableIndexSpace(); - auto sni = runtimeConfig_->serialNumberColumn_; - const auto &serialNumberCol = table.getColumn(sni); - - const auto &addedRows = update.addedRowsIndexSpace(); - const auto &modRowsVec = update.modifiedRowsIndexSpace(); - UInt64Chunk emptyModifiedRows; - const auto *modifiedRowsToUse = sni < modRowsVec.size() ? &modRowsVec[sni] : &emptyModifiedRows; - - auto totalChunkSize = addedRows.size() + modifiedRowsToUse->size(); - if (totalChunkSize == 0) { - return; - } - auto totalChunk = Int64Chunk::create(totalChunkSize); - auto addSlice = totalChunk.take(addedRows.size()); - auto modSlice = totalChunk.drop(addedRows.size()); - - serialNumberCol->fillChunkUnordered(addedRows, &addSlice); - serialNumberCol->fillChunkUnordered(*modifiedRowsToUse, &modSlice); - - serialNumberProcessor_.process(&totalChunk); -} - -std::shared_ptr ImmerProcessor::create(std::shared_ptr userConfig, - std::shared_ptr runtimeConfig) { - auto result = std::make_shared(Private(), std::move(userConfig), - std::move(runtimeConfig)); - std::thread t(&runForever, result); - t.detach(); - return result; -} - -ImmerProcessor::ImmerProcessor(Private,std::shared_ptr userConfig, - std::shared_ptr runtimeConfig) - : userConfig_(std::move(userConfig)), runtimeConfig_(std::move(runtimeConfig)) {} - - ImmerProcessor::~ImmerProcessor() = default; - -void ImmerProcessor::post(ImmerTickingUpdate update) { - mutex_.lock(); - auto empty = todo_.empty(); - todo_.push(std::move(update)); - mutex_.unlock(); - if (empty) { - condvar_.notify_all(); - } -} - -void ImmerProcessor::runForever(std::shared_ptr self) { - std::cerr << "Immer processor thread starting up\n"; - try { - self->runForeverHelper(); - } catch (const std::exception &e) { - streamf(std::cerr, "Caught exception: %o\n", e.what()); - } - std::cerr << "Immer processor thread shutting down\n"; -} - -void ImmerProcessor::runForeverHelper() { - while (true) { - std::unique_lock guard(mutex_); - while (todo_.empty()) { - condvar_.wait(guard); - } - auto update = std::move(todo_.front()); - todo_.pop(); - guard.unlock(); - - processUpdate(update); - } -} - -void ImmerProcessor::processUpdate(const ImmerTickingUpdate &update) { - const auto &table = *update.current(); - auto numCols = table.numColumns(); - auto expectedNumColumns = 2 + userConfig_->numAdditionalCols_; - if (numCols != expectedNumColumns) { - throw std::runtime_error(stringf("Expected %o columns, got %o", expectedNumColumns, numCols)); - } - size_t rowsModifiedEstimated = 0; - size_t cellsModified = 0; - for (const auto &mods : update.modified()) { - rowsModifiedEstimated = std::max(rowsModifiedEstimated, mods->size()); - cellsModified += mods->size(); - } - statsProcessor_.update(update.removed()->size(), - update.added()->size(), rowsModifiedEstimated, cellsModified); - - processSerialNumber(update); - size_t elapsedSeconds; - if (stopwatch_.pulse(&elapsedSeconds)) { - tableScanner_.scanTable(table); - statsProcessor_.showStats(numCols, elapsedSeconds); - fragmentationAnalyzer_.analyze(*update.current()); - } -} - -void ImmerProcessor::processSerialNumber(const ImmerTickingUpdate &update) { - // Do a bunch of work to make sure we received contiguous serial numbers without any skips. - const auto &table = *update.current(); - auto sni = runtimeConfig_->serialNumberColumn_; - const auto &serialNumberCol = table.getColumn(sni); - - const auto &addedRows = update.added(); - const auto &modRowsVec = update.modified(); - auto emptyModifiedRows = RowSequence::createEmpty(); - const auto *modifiedRowsToUse = sni < modRowsVec.size() ? modRowsVec[sni].get() : emptyModifiedRows.get(); - - auto totalChunkSize = addedRows->size() + modifiedRowsToUse->size(); - if (totalChunkSize == 0) { - return; - } - auto totalChunk = Int64Chunk::create(totalChunkSize); - auto addSlice = totalChunk.take(addedRows->size()); - auto modSlice = totalChunk.drop(addedRows->size()); - - serialNumberCol->fillChunk(*addedRows, &addSlice); - serialNumberCol->fillChunk(*modifiedRowsToUse, &modSlice); - - serialNumberProcessor_.process(&totalChunk); -} - -void SerialNumberProcessor::process(Int64Chunk *unorderedSerialNumbers) { - auto &usns = *unorderedSerialNumbers; - std::sort(usns.begin(), usns.end()); - auto firstSerial = *usns.begin(); - auto lastSerial = *(usns.end() - 1); // Safe because chunk is not empty. - if (lastSerial - firstSerial + 1 != usns.size()) { - throw std::runtime_error(stringf("Serial numbers not contiguous [%o..%o]", firstSerial, lastSerial)); - } - if (firstSerial != nextExpectedSerial_) { - throw std::runtime_error(stringf("Expected next serial %o, got %o", nextExpectedSerial_, - firstSerial)); - } - nextExpectedSerial_ = lastSerial + 1; - streamf(std::cout, "next expected serial %o\n", nextExpectedSerial_); -} - -Stopwatch::Stopwatch() { - auto now = std::chrono::steady_clock::now(); - start_ = now; - last_ = now; -} - -bool Stopwatch::pulse(size_t *elapsedSeconds) { - auto now = std::chrono::steady_clock::now(); - *elapsedSeconds = std::chrono::duration_cast(now - start_).count(); - auto thisPulse = std::chrono::duration_cast(now - last_).count(); - if (thisPulse < 5) { - return false; - } - last_ = now; - return true; -} - -void StatsProcessor::update(size_t rowsRemoved, size_t rowsAdded, - size_t rowsModifiedEstimated, size_t cellsModified) { - rowsRemoved_ += rowsRemoved; - rowsAdded_ += rowsAdded; - rowsModifiedEstimated_ += rowsModifiedEstimated; - cellsModified_ += cellsModified; -} - -void StatsProcessor::showStats(size_t numCols, size_t elapsedSeconds) { - auto deltaRemoved = rowsRemoved_ - lastRowsRemoved_; - auto deltaAdded = rowsAdded_ - lastRowsAdded_; - auto deltaModifiedRows = rowsModifiedEstimated_ - lastRowsModifiedEstimated_; - auto deltaModifiedCells = cellsModified_ - lastCellsModified_; - auto totalRows = rowsRemoved_ + rowsAdded_ + rowsModifiedEstimated_; - auto totalCells = rowsRemoved_ * numCols + rowsAdded_ * numCols + cellsModified_; - auto rowsPerSec = totalRows / elapsedSeconds; - auto cellsPerSec = totalCells / elapsedSeconds; - - lastRowsRemoved_ = rowsRemoved_; - lastRowsAdded_ = rowsAdded_; - lastRowsModifiedEstimated_ = rowsModifiedEstimated_; - lastCellsModified_ = cellsModified_; - streamf(std::cout, - "removed %o rows, added %o rows, modified %o rows (%o cells) (totals %o, %o, %o, %o). " - "Total %o rows (%o cells) in %o seconds (%o rows / second) (%o cells / second)\n", - deltaRemoved, deltaAdded, deltaModifiedRows, deltaModifiedCells, - rowsRemoved_, rowsAdded_, rowsModifiedEstimated_, cellsModified_, - totalRows, totalCells, elapsedSeconds, rowsPerSec, cellsPerSec); -} - -void TableScanner::scanTable(const Table &table) { - auto start = std::chrono::system_clock::now(); - auto ncols = table.numColumns(); - auto nrows = table.numRows(); - std::vector results(ncols); - auto allRows = RowSequence::createSequential(0, nrows); - for (size_t colNum = 0; colNum < ncols; ++colNum) { - const auto &col = table.getColumn(colNum); - auto chunk = ChunkMaker::createChunkFor(*col, nrows); - col->fillChunk(*allRows, &chunk.unwrap()); - - // Assume for now that all the columns are int64_t - const auto &typedChunk = chunk.get(); - int64_t total = 0; - for (auto element : typedChunk) { - total += element; - } - results[colNum] = total; - } - auto end = std::chrono::system_clock::now(); - auto duration = std::chrono::duration_cast(end - start).count(); - auto durationMillis = (double)duration / 1'000; - streamf(std::cout, "Table scan took %o millis. Sums are %o\n", durationMillis, - separatedList(results.begin(), results.end(), ", ")); -} - -void FragmentationAnalyzer::analyze(const Table &table) { - for (size_t i = 0; i < table.numColumns(); ++i) { - const auto &col = table.getColumn(i); - auto what = stringf("Col %o", i); - analyzeHelper(what, *col); - } -} - -void FragmentationAnalyzer::analyzeHelper(std::string_view what, const ColumnSource &col) { - const auto underlying = col.backdoor(); - const auto * const *vecpp = std::any_cast*>(&underlying); - if (vecpp == nullptr) { - return; - } - const auto *src = *vecpp; - std::map sizes; - auto analyze = [&sizes](const int64_t *begin, const int64_t *end) { - auto size = end - begin; - ++sizes[size]; - }; - immer::for_each_chunk(src->begin(), src->end(), analyze); - - auto renderSize = [](std::ostream &s, const std::pair &p) { - streamf(s, "%o:%o", p.first, p.second); - }; - - streamf(std::cout, "Immer histogram for %o: %o\n", what, separatedList(sizes.begin(), sizes.end(), - ", ", renderSize)); -} - -UserConfig::UserConfig(std::string server, bool wantLastBy, bool wantImmer, int64_t period, - int64_t tableSize, int64_t endSerialNumber, int64_t keyFactor, int64_t numAdditionalCols) : - server_(std::move(server)), wantLastBy_(wantLastBy), wantImmer_(wantImmer), period_(period), - tableSize_(tableSize),endSerialNumber_(endSerialNumber), keyFactor_(keyFactor), - numAdditionalCols_(numAdditionalCols) {} -UserConfig::~UserConfig() = default; - -std::string getWhat(std::exception_ptr ep) { - try { - std::rethrow_exception(std::move(ep)); - } catch (const std::exception &e) { - return e.what(); - } catch (...) { - return "(unknown exception)"; - } -} - -std::ostream &operator<<(std::ostream &s, const UserConfig &o) { - return streamf(s, "server: %o\n" - "wantLastBy: %o\n" - "wantImmer: %o\n" - "period: %o\n" - "tableSize: %o\n" - "endSerialNumber: %o\n" - "keyFactor: %o\n" - "numAdditionalCols: %o", - o.server_, o.wantLastBy_, o.wantImmer_, o.period_, o.tableSize_, o.endSerialNumber_, o.keyFactor_, - o.numAdditionalCols_); -} -} // namespace diff --git a/cpp-examples/create_table_with_arrow_flight/CMakeLists.txt b/cpp-examples/create_table_with_arrow_flight/CMakeLists.txt index 6269e738a55..25488fa7962 100644 --- a/cpp-examples/create_table_with_arrow_flight/CMakeLists.txt +++ b/cpp-examples/create_table_with_arrow_flight/CMakeLists.txt @@ -6,12 +6,6 @@ set(CMAKE_CXX_STANDARD 17) add_subdirectory(../../cpp-client/deephaven deephaven_dir) #find_package(deephaven REQUIRED) -find_package(Arrow REQUIRED) -find_package(ArrowFlight REQUIRED HINTS ${Arrow_DIR}) -find_package(Protobuf REQUIRED) -find_package(gRPC REQUIRED) -find_package(Threads REQUIRED) - add_executable(create_table_with_arrow_flight main.cc) target_link_libraries(create_table_with_arrow_flight deephaven::client) diff --git a/cpp-examples/create_table_with_arrow_flight/main.cc b/cpp-examples/create_table_with_arrow_flight/main.cc index 94c874df43f..b992d4ff429 100644 --- a/cpp-examples/create_table_with_arrow_flight/main.cc +++ b/cpp-examples/create_table_with_arrow_flight/main.cc @@ -53,7 +53,7 @@ void doit(const TableHandleManager &manager) { okOrThrow(DEEPHAVEN_EXPR_MSG(schemaBuilder.AddField(priceField))); } - // 4. Add "Volume" column (type: int) to schema + // 4. Add "Volume" column (type: int32) to schema { auto volumeMetadata = std::make_shared(); okOrThrow(DEEPHAVEN_EXPR_MSG(volumeMetadata->Set("deephaven:type", "int"))); diff --git a/cpp-examples/create_table_with_table_maker/CMakeLists.txt b/cpp-examples/create_table_with_table_maker/CMakeLists.txt index 1ab8e343f60..01057ca522d 100644 --- a/cpp-examples/create_table_with_table_maker/CMakeLists.txt +++ b/cpp-examples/create_table_with_table_maker/CMakeLists.txt @@ -6,12 +6,6 @@ set(CMAKE_CXX_STANDARD 17) add_subdirectory(../../cpp-client/deephaven deephaven_dir) #find_package(deephaven REQUIRED) -find_package(Arrow REQUIRED) -find_package(ArrowFlight REQUIRED HINTS ${Arrow_DIR}) -find_package(Protobuf REQUIRED) -find_package(gRPC REQUIRED) -find_package(Threads REQUIRED) - add_executable(create_table_with_table_maker main.cc) target_link_libraries(create_table_with_table_maker deephaven::client) diff --git a/cpp-examples/do_exchange/CMakeLists.txt b/cpp-examples/do_exchange/CMakeLists.txt deleted file mode 100644 index 8e00f4a7a9b..00000000000 --- a/cpp-examples/do_exchange/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -cmake_minimum_required(VERSION 3.16) -project(do_exchange) - -set(CMAKE_CXX_STANDARD 17) - - -add_subdirectory(../../cpp-client/deephaven deephaven_dir) -#find_package(deephaven REQUIRED) - -find_package(Arrow REQUIRED) -find_package(ArrowFlight REQUIRED HINTS ${Arrow_DIR}) -find_package(Protobuf REQUIRED) -find_package(gRPC REQUIRED) -find_package(Threads REQUIRED) -find_package(Immer REQUIRED) - -add_executable(do_exchange main.cc) - -target_link_libraries(do_exchange deephaven::client) diff --git a/cpp-examples/do_exchange/main.cc b/cpp-examples/do_exchange/main.cc deleted file mode 100644 index 08817531c55..00000000000 --- a/cpp-examples/do_exchange/main.cc +++ /dev/null @@ -1,556 +0,0 @@ -/* - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -#include -#include -#include -#include - -#include "deephaven/client/client.h" -#include "deephaven/client/ticking.h" -#include "deephaven/client/chunk/chunk_maker.h" -#include "deephaven/client/chunk/chunk.h" -#include "deephaven/client/container/row_sequence.h" -#include "deephaven/client/table/table.h" -#include "deephaven/client/utility/table_maker.h" -#include "deephaven/client/utility/utility.h" -#include "immer/algorithm.hpp" - -using deephaven::client::Client; -using deephaven::client::NumCol; -using deephaven::client::SortPair; -using deephaven::client::TableHandle; -using deephaven::client::TableHandleManager; -using deephaven::client::TickingCallback; -using deephaven::client::ClassicTickingUpdate; -using deephaven::client::ImmerTickingUpdate; -using deephaven::client::chunk::ChunkMaker; -using deephaven::client::chunk::AnyChunk; -using deephaven::client::chunk::ChunkVisitor; -using deephaven::client::container::RowSequence; -using deephaven::client::container::RowSequenceBuilder; -using deephaven::client::chunk::DoubleChunk; -using deephaven::client::chunk::Int32Chunk; -using deephaven::client::chunk::Int64Chunk; -using deephaven::client::chunk::UInt64Chunk; -using deephaven::client::table::Table; -using deephaven::client::utility::makeReservedVector; -using deephaven::client::utility::okOrThrow; -using deephaven::client::utility::separatedList; -using deephaven::client::utility::streamf; -using deephaven::client::utility::stringf; -using deephaven::client::utility::TableMaker; -using deephaven::client::utility::valueOrThrow; - -using std::size_t; - -namespace { -void doit(const TableHandleManager &manager); -void makeModifiesHappen(const TableHandleManager &manager); -void millionRows(const TableHandleManager &manager); -void lastBy(const TableHandleManager &manager); -void varietyOfTypes(const TableHandleManager &manager); -void demo(const TableHandleManager &manager); -template -void ensure(std::vector *vec, size_t size); -template -struct optionalAdaptor_t { - explicit optionalAdaptor_t(const std::optional &value) : value_(value) {} - const std::optional &value_; - - friend std::ostream &operator<<(std::ostream &s, const optionalAdaptor_t &o) { - if (!o.value_.has_value()) { - return s << "[none]"; - } - return s << *o.value_; - } -}; - -template -optionalAdaptor_t adaptOptional(const std::optional &value) { - return optionalAdaptor_t(value); -} -} // namespace - -int main() { - const char *server = "localhost:10000"; - try { - auto client = Client::connect(server); - auto manager = client.getManager(); - varietyOfTypes(manager); - } catch (const std::exception &e) { - std::cerr << "Caught exception: " << e.what() << '\n'; - } -} - -// TODO(kosak): we should standardize on either deephaven or io::deephaven - -namespace { -class Callback final : public TickingCallback { -public: - void onFailure(std::exception_ptr ep) final; - void onTick(const ClassicTickingUpdate &update) final; - void onTick(ImmerTickingUpdate update) final; - - bool failed() const { return failed_; } - -private: - std::atomic failed_ = false; -}; - -class DemoCallback final : public TickingCallback { -public: - void onFailure(std::exception_ptr ep) final; - void onTick(const ClassicTickingUpdate &update) final; - void onTick(ImmerTickingUpdate update) final; - -private: - void processClassicCommon(const Table &table, const UInt64Chunk &affectedRows); - void processImmerCommon(const Table &table, const RowSequence &affectedRows); - void updateCache(const Int64Chunk &tableContentsKeys, const Int64Chunk &tableContentsValues); - void periodicCheck(); - - std::vector> receivedValues_; - std::vector> recalcedValues_; - int64_t maxValueReceived_ = -1; - int64_t nextValueToRecalc_ = 0; -}; - -// or maybe make a stream manipulator -std::string getWhat(std::exception_ptr ep); - -void Callback::onFailure(std::exception_ptr ep) { - streamf(std::cerr, "Callback reported failure: %o\n", getWhat(std::move(ep))); - failed_ = true; -} - -class ElementStreamer final { -public: - ElementStreamer(std::ostream &s, size_t index) : s_(s), index_(index) {} - - template - void operator()(const T &chunk) const { - s_ << chunk.data()[index_]; - } - -private: - std::ostream &s_; - size_t index_ = 0; -}; - -void dumpTable(std::string_view what, const Table &table, const std::vector &whichCols, - std::shared_ptr rows); - -void Callback::onTick(const ClassicTickingUpdate &update) { - streamf(std::cout, "adds: %o\n", *update.addedRowsKeySpace()); - auto render = [](std::ostream &s, const std::shared_ptr &rs) { - s << *rs; - }; - streamf(std::cout, "modifies: %o\n", separatedList(update.modifiedRowsKeySpace().begin(), - update.modifiedRowsKeySpace().end(), " === ", render)); -} - -void Callback::onTick(ImmerTickingUpdate update) { - auto ncols = update.current()->numColumns(); - auto allCols = makeReservedVector(update.current()->numColumns()); - for (size_t i = 0; i < ncols; ++i) { - allCols.push_back(i); - } - dumpTable("removed", *update.beforeRemoves(), allCols, update.removed()); - for (size_t i = 0; i < update.modified().size(); ++i) { - std::vector oneCol{i}; - auto prevText = stringf("Col%o-prev", i); - auto currText = stringf("Col%o-curr", i); - dumpTable(prevText, *update.beforeModifies(), allCols, update.modified()[i]); - dumpTable(currText, *update.current(), allCols, update.modified()[i]); - } - dumpTable("added", *update.current(), allCols, update.added()); -} - -void dumpTable(std::string_view what, const Table &table, const std::vector &whichCols, - std::shared_ptr rows) { - if (rows->empty()) { - return; - } - streamf(std::cout, "===== THIS IS %o =====\n", what); - // Deliberately chosen to be small so I can test chunking. - const size_t chunkSize = 16; - - auto ncols = whichCols.size(); - auto chunks = makeReservedVector(ncols); - for (auto col : whichCols) { - const auto &c = table.getColumn(col); - auto chunk = ChunkMaker::createChunkFor(*c, chunkSize); - chunks.push_back(std::move(chunk)); - } - - while (true) { - auto chunkOfRows = rows->take(chunkSize); - rows = rows->drop(chunkSize); - auto thisSize = chunkOfRows->size(); - if (thisSize == 0) { - break; - } - - for (size_t i = 0; i < ncols; ++i) { - const auto &c = table.getColumn(whichCols[i]); - auto &chunk = chunks[i].unwrap(); - c->fillChunk(*chunkOfRows, &chunk); - } - - for (size_t j = 0; j < thisSize; ++j) { - ElementStreamer es(std::cout, j); - auto chunkAcceptor = [&es](std::ostream &s, const AnyChunk &chunk) { - chunk.visit(es); - }; - std::cout << separatedList(chunks.begin(), chunks.end(), ", ", chunkAcceptor) << '\n'; - } - } -} - -void doit(const TableHandleManager &manager) { - auto start = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()).count(); - auto tt1 = manager - .timeTable(start, 1 * 1'000'000'000L) -// .select("Foo = (ii % 20)", "Bar = ii") -// .select("Foo = ii < 3 ? ii * 10 : (i == 3 ? 15 : (i == 4 ? 12 : 3))" -// // .select("Foo = ii < 10 ? ii * 10 : 23") - .select("Foo = ((ii * 0xdeadbeef) + 0xbaddbabe) % 5000", "Bar = 34.2 + ii", "II = ii", "SomeInt = (int) Bar") - .sort({SortPair::ascending("Foo", false)}) - .head(10) - //.tail(3) // this will give us deletes - ; - //auto tt2 = manager.timeTable(start, 2 * 1'000'000'000L).select("TS2 = Timestamp"); - //auto t = tt1.crossJoin(tt2, std::vector(), {}); - - - // tt1.bindToVariable("showme"); - - // // .select("Foo = (ii % 20)") - // // .select("Foo = ii < 3 ? ii * 10 : (i == 3 ? 15 : (i == 4 ? 12 : 3))") - // // .select("Foo = ii < 10 ? ii * 10 : 23") - // .select("Foo = ((ii * 0xdeadbeef) + 0xbaddbabe) % 5000") - // .sort({SortPair::ascending("Foo", false)}) - // .head(10); - - auto myCallback = std::make_shared(); - auto subscriptionHandle = tt1.subscribe(myCallback, true); - uint32_t tens_of_seconds_to_run = 50000; - while (tens_of_seconds_to_run-- > 0) { - std::this_thread::sleep_for(std::chrono::milliseconds (100)); - if (myCallback->failed()) { - std::cerr << "callback reported failure, aborting in main subscription thread.\n"; - break; - } - } - std::cerr << "I unsubscribed here.\n"; - tt1.unsubscribe(std::move(subscriptionHandle)); - std::this_thread::sleep_for(std::chrono::seconds(5)); - std::cerr << "exiting.\n"; -} - -// let's keep this off to the side -// auto tLeft = manager -// .emptyTable(10) -// .select("II = ii", "Zamboni = `zamboni`+ii"); -// auto tRight = manager -// .timeTable(start, 1 * 1'000'000'000L) -// .update("II = 0L") -// .tail(1); -// auto tt1 = tLeft.naturalJoin(tRight, {"II"}, {}); - - -// also this -// auto tt1 = manager -// .timeTable(start, 1 * 1'000'000'000L) -// // .select("Foo = (ii % 20)") -// // .select("Foo = ii < 3 ? ii * 10 : (i == 3 ? 15 : (i == 4 ? 12 : 3))") -// // .select("Foo = ii < 10 ? ii * 10 : 23") -// .select("Foo = ((ii * 0xdeadbeef) + 0xbaddbabe) % 5000") -// .sort({SortPair::ascending("Foo", false)}) -// .head(10); -//.tail(3); // this will give us deletes -//auto tt2 = manager.timeTable(start, 2 * 1'000'000'000L).select("TS2 = Timestamp"); -//auto t = tt1.crossJoin(tt2, std::vector(), {}); - - -// tt1.bindToVariable("showme"); - -// // .select("Foo = (ii % 20)") -// // .select("Foo = ii < 3 ? ii * 10 : (i == 3 ? 15 : (i == 4 ? 12 : 3))") -// // .select("Foo = ii < 10 ? ii * 10 : 23") -// .select("Foo = ((ii * 0xdeadbeef) + 0xbaddbabe) % 5000") -// .sort({SortPair::ascending("Foo", false)}) -// .head(10); - - -void makeModifiesHappen(const TableHandleManager &manager) { - auto start = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()).count(); - - auto tLeft = manager - .emptyTable(10) - .select("II = ii", "Zamboni = ii"); - auto tRight = manager - .timeTable(start, 1 * 1'000'000'000L) - .select("II = 0L", "TS = (long)(Timestamp.getNanos()/1000000000)") - .tail(1); - auto tt1 = tLeft.naturalJoin(tRight, {"II"}, {}).select("II", "Zamboni", "TS"); - - tt1.bindToVariable("showme"); - - auto myCallback = std::make_shared(); - auto handle = tt1.subscribe(myCallback, true); - std::this_thread::sleep_for(std::chrono::seconds(5'000)); - std::cerr << "I unsubscribed here\n"; - tt1.unsubscribe(std::move(handle)); - std::this_thread::sleep_for(std::chrono::seconds(5)); - std::cerr << "exiting\n"; -} - -void millionRows(const TableHandleManager &manager) { - auto start = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()).count(); - -// const size_t topAndBottomSize = 500'000; - const size_t topAndBottomSize = 10; - auto tTop = manager.emptyTable(topAndBottomSize).select("Value = ii"); - auto tBottom = manager.emptyTable(topAndBottomSize).select("Value = 10_000_000 + ii"); - auto pulsatingMiddle = manager.timeTable(start, 1 * 1'000'000'000L) - .tail(10) - .select("Selector = ((int)(Timestamp.getNanos() / 1_000_000_000)) % 20") - .where("Selector < 10") - .select("Value = 1_000_000L + Selector"); - - auto table = tTop.merge({pulsatingMiddle, tBottom}); - - table.bindToVariable("showme"); - - auto myCallback = std::make_shared(); - auto handle = table.subscribe(myCallback, true); - std::this_thread::sleep_for(std::chrono::seconds(5'000)); - std::cerr << "I unsubscribed here\n"; - table.unsubscribe(handle); - std::this_thread::sleep_for(std::chrono::seconds(5)); - std::cerr << "exiting\n"; -} - -void demo(const TableHandleManager &manager) { - auto start = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()).count(); - - const long modSize = 1000; - auto table = manager.timeTable(start, 1 * 100'000'000L) - .view("II = ii") - .where("II < 500") - .view("Temp1 = (II ^ (long)(II / 65536)) * 0x8febca6b", - "Temp2 = (Temp1 ^ ((long)(Temp1 / 8192))) * 0xc2b2ae35", - "HashValue = Temp2 ^ (long)(Temp2 / 65536)", - "II"); - - // might as well use this interface once in a while - auto [hv, ii] = table.getCols("HashValue", "II"); - auto t2 = table.view((hv % modSize).as("Key"), "Value = II"); - auto key = t2.getNumCol("Key"); - auto lb = t2.lastBy(key).sort({key.ascending()}); - - lb.bindToVariable("showme"); - - auto myCallback = std::make_shared(); - auto handle = lb.subscribe(myCallback, false); - std::this_thread::sleep_for(std::chrono::seconds(5'000)); - std::cerr << "I unsubscribed here\n"; - lb.unsubscribe(handle); - std::this_thread::sleep_for(std::chrono::seconds(5)); - std::cerr << "exiting\n"; -} - -void lastBy(const TableHandleManager &manager) { - auto start = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()).count(); - - const long modSize = 10; - auto table = manager.timeTable(start, 1 * 10'000'000L) - .select("II = ii") - .where("II < 2000") - .select("Temp1 = (II ^ (long)(II / 65536)) * 0x8febca6b", - "Temp2 = (Temp1 ^ ((long)(Temp1 / 8192))) * 0xc2b2ae35", - "HashValue = Temp2 ^ (long)(Temp2 / 65536)", - "II"); - - // might as well use this interface once in a while - auto [hv, ii] = table.getCols("HashValue", "II"); - auto t2 = table.select((hv % modSize).as("Key"), "Value = II"); - auto key = t2.getNumCol("Key"); - auto lb = t2.lastBy(key).sort({key.ascending()}); - - lb.bindToVariable("showme"); - - auto myCallback = std::make_shared(); - auto handle = lb.subscribe(myCallback, true); - std::this_thread::sleep_for(std::chrono::seconds(5'000)); - std::cerr << "I unsubscribed here\n"; - lb.unsubscribe(handle); - std::this_thread::sleep_for(std::chrono::seconds(5)); - std::cerr << "exiting\n"; -} - -void varietyOfTypes(const TableHandleManager &manager) { - auto start = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()).count(); - - auto table = manager.timeTable(start, 1 * 1'000'000'000L) - .select("XXX = `hello`", "YYY = (short)12", "ZZZ = (byte)65", "QQ = `true`"); - - table.bindToVariable("showme"); - - auto myCallback = std::make_shared(); - auto handle = table.subscribe(myCallback, true); - std::this_thread::sleep_for(std::chrono::seconds(8)); - std::cerr << "I unsubscribed here\n"; - table.unsubscribe(handle); - std::this_thread::sleep_for(std::chrono::seconds(5)); - std::cerr << "exiting\n"; -} - -void DemoCallback::onFailure(std::exception_ptr ep) { - streamf(std::cerr, "Callback reported failure: %o\n", getWhat(std::move(ep))); -} - -void DemoCallback::onTick(const ClassicTickingUpdate &update) { - const auto &table = *update.currentTableIndexSpace(); - const auto &addedRows = update.addedRowsIndexSpace(); - if (table.numColumns() != 2) { - throw std::runtime_error(stringf("Expected 2 columns, got %o", table.numColumns())); - } - processClassicCommon(table, addedRows); - - const auto &modifiedRows = update.modifiedRowsIndexSpace(); - if (!modifiedRows.empty()) { - if (modifiedRows.size() != 2) { - throw std::runtime_error(stringf("Expected 2 modified rows, got %o", modifiedRows.size())); - } - const auto &tableContentsKeyMods = modifiedRows[0]; - const auto &tableContenstsValueMods = modifiedRows[1]; - if (tableContentsKeyMods.size() != 0) { - throw std::runtime_error( - stringf("Our application doesn't modify the key column, but got %o", - tableContentsKeyMods.size())); - } - processClassicCommon(table, tableContenstsValueMods); - } - periodicCheck(); -} - -void DemoCallback::processClassicCommon(const Table &table, const UInt64Chunk &affectedRows) { - auto nrows = affectedRows.size(); - auto tableContentsKeys = Int64Chunk::create(nrows); - auto tableContentsValues = Int64Chunk::create(nrows); - const auto &keyCol = table.getColumn(0); - const auto &valueCol = table.getColumn(1); - keyCol->fillChunkUnordered(affectedRows, &tableContentsKeys); - valueCol->fillChunkUnordered(affectedRows, &tableContentsValues); - updateCache(tableContentsKeys, tableContentsValues); -} - -void DemoCallback::onTick(ImmerTickingUpdate update) { - const auto &table = *update.current(); - const auto &added = *update.added(); - if (table.numColumns() != 2) { - throw std::runtime_error(stringf("Expected 2 columns, got %o", table.numColumns())); - } - processImmerCommon(table, added); - - const auto &modified = update.modified(); - if (!modified.empty()) { - if (modified.size() != 2) { - throw std::runtime_error(stringf("Expected 2 modified rows, got %o", modified.size())); - } - const auto &tableContentsKeyMods = *modified[0]; - const auto &tableContenstsValueMods = *modified[1]; - if (!tableContentsKeyMods.empty()) { - throw std::runtime_error(stringf("Our application doesn't modify the key column, but got %o", - tableContentsKeyMods.size())); - } - processImmerCommon(table, tableContenstsValueMods); - } - periodicCheck(); -} - -void DemoCallback::processImmerCommon(const Table &table, const RowSequence &affectedRows) { - auto nrows = affectedRows.size(); - auto tableContentsKeys = Int64Chunk::create(nrows); - auto tableContentsValues = Int64Chunk::create(nrows); - const auto &keyCol = table.getColumn(0); - const auto &valueCol = table.getColumn(1); - keyCol->fillChunk(affectedRows, &tableContentsKeys); - valueCol->fillChunk(affectedRows, &tableContentsValues); - updateCache(tableContentsKeys, tableContentsValues); -} - -void DemoCallback::updateCache(const Int64Chunk &tableContentsKeys, const Int64Chunk &tableContentsValues) { - auto size = tableContentsKeys.size(); - if (size != tableContentsValues.size()) { - throw std::runtime_error( - stringf("Expected tableContentsKeys.size() == tableContentsValues.size(), got (%o != %o)", - size, tableContentsValues.size())); - } - for (size_t i = 0; i < size; ++i) { - auto key = tableContentsKeys.data()[i]; - auto value = tableContentsValues.data()[i]; - streamf(std::cout, "%o - %o\n", key, value); - ensure(&receivedValues_, key + 1); - receivedValues_[key] = value; - maxValueReceived_ = std::max(maxValueReceived_, value); - } -} - -void DemoCallback::periodicCheck() { - streamf(std::cout, "hello, max value seen is %o\n", maxValueReceived_); - - auto end = maxValueReceived_ + 1; - for (auto ii = nextValueToRecalc_; ii != end; ++ii) { - // We need to do these divides as floating point because that's what the server is doing. - auto temp1 = (ii ^ (int64_t)(ii / 65536.0)) * 0x8febca6b; - auto temp2 = (temp1 ^ ((int64_t)(temp1 / 8192.0))) * 0xc2b2ae35; - auto hashValue = temp2 ^ (int64_t)(temp2 / 65536.0); - auto key = hashValue % 1000; - auto value = ii; - ensure(&recalcedValues_, key + 1); - recalcedValues_[key] = value; - } - - if (receivedValues_.size() != recalcedValues_.size()) { - throw std::runtime_error(stringf("receivedValues_.size() != recalcedValues_.size() (%o != %o)", - receivedValues_.size(), recalcedValues_.size())); - } - - for (auto ii = nextValueToRecalc_; ii != end; ++ii) { - if (receivedValues_[ii] == recalcedValues_[ii]) { - continue; - } - throw std::runtime_error(stringf("At offset %o, received != recalc (%o != %o)", - ii, adaptOptional(receivedValues_[ii]), adaptOptional(recalcedValues_[ii]))); - } - - nextValueToRecalc_ = end; -} - -template -void ensure(std::vector *vec, size_t size) { - if (size > vec->size()) { - vec->resize(size); - } -} - -std::string getWhat(std::exception_ptr ep) { - try { - std::rethrow_exception(std::move(ep)); - } catch (const std::exception &e) { - return e.what(); - } catch (...) { - return "(unknown exception)"; - } -} -} // namespace diff --git a/cpp-examples/hello_world/CMakeLists.txt b/cpp-examples/hello_world/CMakeLists.txt index 6e021454316..594c2a39004 100644 --- a/cpp-examples/hello_world/CMakeLists.txt +++ b/cpp-examples/hello_world/CMakeLists.txt @@ -6,12 +6,6 @@ set(CMAKE_CXX_STANDARD 17) add_subdirectory(../../cpp-client/deephaven deephaven_dir) #find_package(deephaven REQUIRED) -find_package(Arrow REQUIRED) -find_package(ArrowFlight REQUIRED HINTS ${Arrow_DIR}) -find_package(Protobuf REQUIRED) -find_package(gRPC REQUIRED) -find_package(Threads REQUIRED) - add_executable(hello_world main.cc) target_link_libraries(hello_world deephaven::client) diff --git a/cpp-examples/read_csv/CMakeLists.txt b/cpp-examples/read_csv/CMakeLists.txt new file mode 100644 index 00000000000..44a88f9ae92 --- /dev/null +++ b/cpp-examples/read_csv/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required(VERSION 3.16) +project(read_csv) + +set(CMAKE_CXX_STANDARD 17) + +add_subdirectory(../../cpp-client/deephaven deephaven_dir) +#find_package(deephaven REQUIRED) + +add_executable(read_csv main.cc) + +target_link_libraries(read_csv deephaven::client) diff --git a/cpp-examples/readcsv/main.cc b/cpp-examples/read_csv/main.cc similarity index 100% rename from cpp-examples/readcsv/main.cc rename to cpp-examples/read_csv/main.cc diff --git a/cpp-examples/read_csv/test.csv b/cpp-examples/read_csv/test.csv new file mode 100644 index 00000000000..657cce95db2 --- /dev/null +++ b/cpp-examples/read_csv/test.csv @@ -0,0 +1,2 @@ +A,B,C +1,2.2,"three" diff --git a/cpp-examples/read_table_with_arrow_flight/CMakeLists.txt b/cpp-examples/read_table_with_arrow_flight/CMakeLists.txt index f2c7b7f3d98..1be006b247c 100644 --- a/cpp-examples/read_table_with_arrow_flight/CMakeLists.txt +++ b/cpp-examples/read_table_with_arrow_flight/CMakeLists.txt @@ -7,12 +7,6 @@ set(CMAKE_CXX_STANDARD 17) add_subdirectory(../../cpp-client/deephaven deephaven_dir) #find_package(deephaven REQUIRED) -find_package(Arrow REQUIRED) -find_package(ArrowFlight REQUIRED HINTS ${Arrow_DIR}) -find_package(Protobuf REQUIRED) -find_package(gRPC REQUIRED) -find_package(Threads REQUIRED) - add_executable(read_table_with_arrow_flight main.cc) target_link_libraries(read_table_with_arrow_flight deephaven::client) diff --git a/cpp-examples/readcsv/CMakeLists.txt b/cpp-examples/readcsv/CMakeLists.txt deleted file mode 100644 index f3fcb96daf0..00000000000 --- a/cpp-examples/readcsv/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -cmake_minimum_required(VERSION 3.16) -project(readcsv) - -set(CMAKE_CXX_STANDARD 17) - -add_subdirectory(../../cpp-client/deephaven deephaven_dir) -#find_package(deephaven REQUIRED) - -find_package(Arrow REQUIRED) -find_package(ArrowFlight REQUIRED HINTS ${Arrow_DIR}) -find_package(Protobuf REQUIRED) -find_package(gRPC REQUIRED) -find_package(Threads REQUIRED) - -add_executable(readcsv main.cc) - -target_link_libraries(readcsv deephaven::client)